Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_kafka_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-7l98l-qqmbg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8fa3f6359d74f7f8846946e2a1039ec5f4bf97c8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-7l98l" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=80da174b-afbe-45e9-b583-c309dae1cfa2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-7l98l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-236df335481f9578f70eb859f68d5ceead3aa27f6c9385fda1ec4c08661c0305 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r pingcap_tiflow_pull_cdc_integration_kafka_test_1870-7l98l GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.0' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build1208120846=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-1870-7l98l-npn2r bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 51079 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 4a470390372c36dc (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 17 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 08:41:23 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11023/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [scripts/download-integration-test-binaries.sh, tests/integration_tests/sequence/run.sh] [Pipeline] echo diff file not matched: scripts/download-integration-test-binaries.sh [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-e9f061d) 203651072 bytes in 1.53 secs (133013202 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/.git/ .git HEAD is now at e9f061d7a enable sequence test POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/11023/head -> origin/pr/11023/head Previous HEAD position was e9f061d7a enable sequence test HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..e9f061d7a Fast-forward scripts/download-integration-test-binaries.sh | 4 ++-- tests/integration_tests/sequence/run.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) ๐Ÿงพ Pre-merged result: e9f061d7a0898b066e3c6e271294672cae36f6d6 e9f061d7a enable sequence test be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-be15534-e9f061d already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1426 0 --:--:-- --:--:-- --:--:-- 1464 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 335 0 --:--:-- --:--:-- --:--:-- 336 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1738 0 --:--:-- --:--:-- --:--:-- 1782 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 542 0 --:--:-- --:--:-- --:--:-- 546 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/659f32a81300f9dbcea9032b3c8e4825555ccfd1/centos7/tidb-server.tar.gz 2024-05-06 16:41:46 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/659f32a81300f9dbcea9032b3c8e4825555ccfd1/centos7/tidb-server.tar.gz [536598984/536598984] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-06 16:41:59 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz 2024-05-06 16:42:22 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz [919298452/919298452] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz 2024-05-06 16:42:52 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz [456051121/456051121] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 16:42:57 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 16:43:00 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 16:43:01 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 16:43:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz 2024-05-06 16:43:03 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz [55896256/55896256] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 16:43:19 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 16:43 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:43 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:43 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 16:43 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 15:59 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 16:43 . drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:43 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:43 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 16:43 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 15:59 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.1.5-2-geae7b5d Git Commit Hash: eae7b5d149e30a02a04ec6116e08b0a8f07fe4aa Git Branch: disable-cgo-for-sync-diff UTC Build Time: 2024-05-06 08:40:40 Go Version: go1.22.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K May 6 16:43 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:43 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:43:22" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading go.uber.org/zap v1.27.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading go.uber.org/mock v0.4.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/emirpasic/gods v1.18.1 go: downloading k8s.io/api v0.28.6 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading modernc.org/mathutil v1.6.0 go: downloading modernc.org/memory v1.7.2 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:40" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:47" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:57" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:57" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:57" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:57" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:57" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-22-ge9f061d7a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 08:45:57" -X "github.com/pingcap/tiflow/pkg/version.GitHash=e9f061d7a0898b066e3c6e271294672cae36f6d6" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-22-ge9f061d7a"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x 2 jenkins jenkins 4.0K May 6 16:51 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:43 .. -rwxr-xr-x 1 jenkins jenkins 220M May 6 16:51 cdc -rwxr-xr-x 1 jenkins jenkins 359M May 6 16:50 cdc.test -rwxr-xr-x 1 jenkins jenkins 183M May 6 16:46 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 183M May 6 16:46 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 16:46 cdc_storage_consumer -rwxr-xr-x 1 jenkins jenkins 12M May 6 16:46 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-22-ge9f061d7a Git Commit Hash: e9f061d7a0898b066e3c6e271294672cae36f6d6 Git Branch: HEAD UTC Build Time: 2024-05-06 08:45:57 Go Version: go version go1.21.0 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-test/rev-be15534-e9f061d) 1191601664 bytes in 36.13 secs (32984978 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 16:51 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:43 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 6 16:51 bin -rwxr-xr-x 1 jenkins jenkins 220M May 6 16:51 cdc -rwxr-xr-x 1 jenkins jenkins 359M May 6 16:50 cdc.test -rwxr-xr-x 1 jenkins jenkins 183M May 6 16:46 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 183M May 6 16:46 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 16:46 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 etc -rwxr-xr-x 1 jenkins jenkins 17M May 6 16:51 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:51 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 16:51 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 16:51 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K May 6 16:51 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 16:51 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K May 6 16:51 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 16:51 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 16:51 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 16:51 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 16:51 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M May 6 16:51 minio -rwxr-xr-x 1 jenkins jenkins 12M May 6 16:46 oauth2-server -rwxr-xr-x 1 jenkins jenkins 37M May 6 16:51 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:51 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 6 16:51 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:51 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 6 16:51 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 6 16:51 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 6 16:51 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:51 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 share -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:51 stores-dump -rwxr-xr-x 1 jenkins jenkins 133M May 6 16:51 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 16:51 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 16:51 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 16:51 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 6 16:51 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 71.09 secs (51553722 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73 [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr [Pipeline] podTemplate Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-pp21w-qtb3b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8f6faf454cd433ec82a6d94094d94c58a3f043dd" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-pp21w" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] { Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] node [Pipeline] podTemplate [Pipeline] { [Pipeline] checkout [Pipeline] node The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6 [Pipeline] podTemplate [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5be6ca2d; decorates RemoteLauncher[hudson.remoting.Channel@478741d7:JNLP4-connect connection from 10.233.84.86/10.233.84.86:55458] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4jxkt-ztp7w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a0dae005af36bfe600fa49cde2ac14c907bc83cd" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4jxkt" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-k4x4v-8nl79 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ca98a9821084b454093bb36ab18aa9a4522f30fd" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-k4x4v" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-cq1g3-tmkd8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "496323b08d4fa840f65a1c3c4b3ecd5f164cae5b" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-cq1g3" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-x9nnr-qbznt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f951a916dccc8cd807e144039219ad1d997e4f74" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-x9nnr" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-j10vp-4xwcr --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "86b679e88ca4d7ced67ebce984b83789a40408aa" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-j10vp" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-t2mdb-7n623 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b60ed5ac52d25098d4fc104b210e66f994e57a9c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-t2mdb" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dg389-kr2r5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3aed97b4c6a7a3c099734ed76514e6c18da0851a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dg389" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-0v21p-70mdn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e7cb5020caf93e98050c4bee26f9fc3005a6cb77" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-0v21p" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 10.12 secs (362282416 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] { [Pipeline] { [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9 The recommended git tool is: git [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-n41ng-p96k5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-676wd-rqd8x [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-chmlt-9j9zf Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-vgp9b-vn9n7 [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // container [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@22b09be2; decorates RemoteLauncher[hudson.remoting.Channel@7704110e:JNLP4-connect connection from 10.233.90.124/10.233.90.124:59442] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@24464c69; decorates RemoteLauncher[hudson.remoting.Channel@54793497:JNLP4-connect connection from 10.233.88.49/10.233.88.49:53262] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=fafa7b5e-6529-4c33-8912-3c5ca59040b2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-pp21w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6 pingcap_tiflow_pull_cdc_integration_kafka_test_1870-pp21w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:53:25 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7730371d; decorates RemoteLauncher[hudson.remoting.Channel@2cade6e6:JNLP4-connect connection from 10.233.107.63/10.233.107.63:56928] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@194644cb; decorates RemoteLauncher[hudson.remoting.Channel@614d2164:JNLP4-connect connection from 10.233.108.125/10.233.108.125:41426] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@efc9589; decorates RemoteLauncher[hudson.remoting.Channel@285beb1d:JNLP4-connect connection from 10.233.93.142/10.233.93.142:52744] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1d4b78e4; decorates RemoteLauncher[hudson.remoting.Channel@72c8f962:JNLP4-connect connection from 10.233.73.177/10.233.73.177:41850] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1022f3c6; decorates RemoteLauncher[hudson.remoting.Channel@4e032c80:JNLP4-connect connection from 10.233.69.96/10.233.69.96:58252] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@28b8d05; decorates RemoteLauncher[hudson.remoting.Channel@2df3f75b:JNLP4-connect connection from 10.233.97.121/10.233.97.121:33620] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] { Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withEnv [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] container [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] cache Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:53:28 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dqtbf-mk6d4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "373fc06b60e7e955d4ffbd5a94e2eef6828e8618" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dqtbf" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: kafka... <<================= kafka downstream isn't support syncpoint record [Mon May 6 16:53:32 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4z7q7-3vx8m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ac52ce04d423c25266cf74d5653d74f892515b29" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4z7q7" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4tqk4-h90wj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4c1cc4b9a91e96e6c70b18a455f549b8b9c4742c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4tqk4" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-mvw62-75t7c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e6780f90ec8634e956baa6c1730ff2b6373a5953" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-mvw62" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:53:35 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1870-2v9k0-2qtlq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0baa0fa8eab369685c12a39f863fe989970dac11" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1870-2v9k0" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1870-n41ng-p96k5โ€™ is offline Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1870-676wd-rqd8xโ€™ is offline Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1870-chmlt-9j9zfโ€™ is offline Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1870-vgp9b-vn9n7โ€™ is offline find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:53:38 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 17.12 secs (214090710 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347043340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:1707, start at 2024-05-06 16:53:56.074047353 +0800 CST m=+5.415917468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:55:56.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:53:56.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:43:56.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347043340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:1707, start at 2024-05-06 16:53:56.074047353 +0800 CST m=+5.415917468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:55:56.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:53:56.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:43:56.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347043e4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:1798, start at 2024-05-06 16:53:56.098044001 +0800 CST m=+5.381805317 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:55:56.106 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:53:56.089 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:43:56.089 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 16:54:01 CST 2024] <<<<<< START cdc server in kafka_big_messages case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages.32633265.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:54:04 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f265c21-ba1a-4c54-9a7f-79fd95f41c36 {"id":"3f265c21-ba1a-4c54-9a7f-79fd95f41c36","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1be5cfc9 3f265c21-ba1a-4c54-9a7f-79fd95f41c36 /tidb/cdc/default/default/upstream/7365807183279631602 {"id":7365807183279631602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f265c21-ba1a-4c54-9a7f-79fd95f41c36 {"id":"3f265c21-ba1a-4c54-9a7f-79fd95f41c36","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1be5cfc9 3f265c21-ba1a-4c54-9a7f-79fd95f41c36 /tidb/cdc/default/default/upstream/7365807183279631602 {"id":7365807183279631602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f265c21-ba1a-4c54-9a7f-79fd95f41c36 {"id":"3f265c21-ba1a-4c54-9a7f-79fd95f41c36","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985641} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1be5cfc9 3f265c21-ba1a-4c54-9a7f-79fd95f41c36 /tidb/cdc/default/default/upstream/7365807183279631602 {"id":7365807183279631602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 627b901d-76dc-414b-82c6-292458ac4878 Info: {"upstream_id":7365807183279631602,"namespace":"default","id":"627b901d-76dc-414b-82c6-292458ac4878","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-06T16:54:04.390469744+08:00","start_ts":449573195886100485,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573195886100485,"checkpoint_ts":449573195886100485,"checkpoint_time":"2024-05-06 16:54:01.045"} [Mon May 6 16:54:04 CST 2024] <<<<<< START kafka consumer in kafka_big_messages case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 15.20 secs (241126667 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min The recommended git tool is: git [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1884cfb9; decorates RemoteLauncher[hudson.remoting.Channel@1e30642a:JNLP4-connect connection from 10.233.106.158/10.233.106.158:43454] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e6113ca; decorates RemoteLauncher[hudson.remoting.Channel@49d7643b:JNLP4-connect connection from 10.233.70.75/10.233.70.75:36104] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5f1c5c4; decorates RemoteLauncher[hudson.remoting.Channel@53152be8:JNLP4-connect connection from 10.233.67.34/10.233.67.34:52686] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@26b4aa36; decorates RemoteLauncher[hudson.remoting.Channel@5a9b7a9c:JNLP4-connect connection from 10.233.72.73/10.233.72.73:50648] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@32f82324; decorates RemoteLauncher[hudson.remoting.Channel@6b3de685:JNLP4-connect connection from 10.233.68.9/10.233.68.9:41466] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 table kafka_big_messages.test not exists for 1-th check, retry later > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 table kafka_big_messages.test exists check diff failed 1-th time, retry later Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:54:16 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 11.33 secs (323371840 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] // timeout [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kafka_compression Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 17.66 secs (207550404 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] // timeout [Pipeline] } + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] // container [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=0cec0fe4-969a-4520-ae76-5c60d8483dbc BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4jxkt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73 pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4jxkt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:54:42 CST 2024] <<<<<< run test case http_api success! >>>>>> [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347303f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:5587, start at 2024-05-06 16:54:41.182698394 +0800 CST m=+5.235724804 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:41.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:41.150 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:41.150 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347303f80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:5587, start at 2024-05-06 16:54:41.182698394 +0800 CST m=+5.235724804 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:41.189 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:41.150 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:41.150 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347303fc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:5659, start at 2024-05-06 16:54:41.15875763 +0800 CST m=+5.163668455 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:56:41.165 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:54:41.151 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:44:41.151 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 16:54:46 CST 2024] <<<<<< START cdc server in kafka_compression case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.70317033.out server --log-file /tmp/tidb_cdc_test/kafka_compression/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_compression/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:54:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4 {"id":"d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985686} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1c9ab1cc d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4 /tidb/cdc/default/default/upstream/7365807377198123522 {"id":7365807377198123522,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4 {"id":"d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985686} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1c9ab1cc d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4 /tidb/cdc/default/default/upstream/7365807377198123522 {"id":7365807377198123522,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4 {"id":"d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985686} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1c9ab1cc d12333fd-ec0b-47d6-9e0c-31bd5c69c9f4 /tidb/cdc/default/default/upstream/7365807377198123522 {"id":7365807377198123522,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7092.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449573208640716802 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573208640716802 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7130.out cli changefeed create --start-ts=449573208640716802 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip' -c gzip Create changefeed successfully! ID: gzip Info: {"upstream_id":7365807377198123522,"namespace":"default","id":"gzip","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=gzip","create_time":"2024-05-06T16:54:51.606678799+08:00","start_ts":449573208640716802,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573208640716802,"checkpoint_ts":449573208640716802,"checkpoint_time":"2024-05-06 16:54:49.700"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:54:52 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> + set +x [Mon May 6 16:54:53 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 16:54:51.562 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:51.602 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:51.723 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:51.733 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:52.699 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:52.708 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 16:54:51.562 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:51.602 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:51.723 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:51.733 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:52.699 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 16:54:52.708 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]") table test.gzip_finish_mark not exists for 1-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: kafka... <<================= table test.gzip_finish_mark not exists for 2-th check, retry later table test.gzip_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7279.out cli changefeed pause -c gzip PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:54:59 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7314.out cli changefeed remove -c gzip Changefeed remove successfully. ID: gzip CheckpointTs: 449573210816512006 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7346.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449573212022374406 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573212022374406 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7384.out cli changefeed create --start-ts=449573212022374406 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy' -c snappy Create changefeed successfully! ID: snappy Info: {"upstream_id":7365807377198123522,"namespace":"default","id":"snappy","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=snappy","create_time":"2024-05-06T16:55:04.498351387+08:00","start_ts":449573212022374406,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573212022374406,"checkpoint_ts":449573212022374406,"checkpoint_time":"2024-05-06 16:55:02.600"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 16:55:05 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 16:55:04.462 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:04.494 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:04.598 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:04.608 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:05.597 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:05.605 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 16:55:04.462 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:04.494 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:04.598 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:04.608 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:05.597 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 16:55:05.605 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]") table test.snappy_finish_mark not exists for 1-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.snappy_finish_mark not exists for 2-th check, retry later table test.snappy_finish_mark not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.snappy_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7511.out cli changefeed pause -c snappy PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7549.out cli changefeed remove -c snappy Changefeed remove successfully. ID: snappy CheckpointTs: 449573214748672008 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7584.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 32.39 secs (113141105 bytes/sec) [Pipeline] { [Pipeline] cache + set +x + tso='449573215915212806 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573215915212806 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7616.out cli changefeed create --start-ts=449573215915212806 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4' -c lz4 Create changefeed successfully! ID: lz4 Info: {"upstream_id":7365807377198123522,"namespace":"default","id":"lz4","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=lz4","create_time":"2024-05-06T16:55:19.40214492+08:00","start_ts":449573215915212806,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573215915212806,"checkpoint_ts":449573215915212806,"checkpoint_time":"2024-05-06 16:55:17.450"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Mon May 6 16:55:20 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 16:55:19.364 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:19.398 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:19.499 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:19.508 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:20.499 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:20.507 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 16:55:19.364 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:19.398 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:19.499 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:19.508 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:20.499 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 16:55:20.507 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]") table test.lz4_finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.lz4_finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.lz4_finish_mark exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3475b2100019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:1718, start at 2024-05-06 16:55:25.106093298 +0800 CST m=+5.379208413 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:25.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:25.110 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:25.110 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7754.out cli changefeed pause -c lz4 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3475b2100019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:1718, start at 2024-05-06 16:55:25.106093298 +0800 CST m=+5.379208413 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:25.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:25.110 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:25.110 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3475b3940013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:1797, start at 2024-05-06 16:55:25.186345777 +0800 CST m=+5.398723847 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:25.193 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:25.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:25.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7787.out cli changefeed remove -c lz4 Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Changefeed remove successfully. ID: lz4 CheckpointTs: 449573216937574429 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7825.out cli tso query --pd=http://127.0.0.1:2379 The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449573219296870404 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573219296870404 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7861.out cli changefeed create --start-ts=449573219296870404 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd' -c zstd Create changefeed successfully! ID: zstd Info: {"upstream_id":7365807377198123522,"namespace":"default","id":"zstd","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=zstd","create_time":"2024-05-06T16:55:32.248543368+08:00","start_ts":449573219296870404,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573219296870404,"checkpoint_ts":449573219296870404,"checkpoint_time":"2024-05-06 16:55:30.350"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting TLS TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Mon May 6 16:55:33 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 16:55:32.214 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:32.244 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:32.349 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:32.357 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:33.349 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:33.356 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 16:55:32.214 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:32.244 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:32.349 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:32.357 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:33.349 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 16:55:33.356 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]") table test.zstd_finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 15.31 secs (239410205 bytes/sec) [Pipeline] { [Pipeline] cache table test.zstd_finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.zstd_finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.zstd_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7991.out cli changefeed pause -c zstd PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347692a40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:3564, start at 2024-05-06 16:55:39.457781436 +0800 CST m=+6.022420068 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:57:39.467 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:55:39.433 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:45:39.433 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3629.out cli tso query --pd=https://127.0.0.1:2579 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8026.out cli changefeed remove -c zstd Changefeed remove successfully. ID: zstd CheckpointTs: 449573220306124831 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd PASS + set +x + tso='449573222333284353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573222333284353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:55:43 CST 2024] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.36843686.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:55:46 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:55:48 GMT < Content-Length: 1233 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae2328eb-b077-4c1a-9f71-6865b24dbde1 {"id":"ae2328eb-b077-4c1a-9f71-6865b24dbde1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985744} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d1d80798b ae2328eb-b077-4c1a-9f71-6865b24dbde1 /tidb/cdc/default/default/upstream/7365807625120241797 {"id":7365807625120241797,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae2328eb-b077-4c1a-9f71-6865b24dbde1 {"id":"ae2328eb-b077-4c1a-9f71-6865b24dbde1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985744} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d1d80798b ae2328eb-b077-4c1a-9f71-6865b24dbde1 /tidb/cdc/default/default/upstream/7365807625120241797 {"id":7365807625120241797,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae2328eb-b077-4c1a-9f71-6865b24dbde1 {"id":"ae2328eb-b077-4c1a-9f71-6865b24dbde1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985744} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d1d80798b ae2328eb-b077-4c1a-9f71-6865b24dbde1 /tidb/cdc/default/default/upstream/7365807625120241797 {"id":7365807625120241797,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3751.out cli changefeed create --start-ts=449573222333284353 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-8436?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365807625120241797,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-8436?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:55:51.132577511+08:00","start_ts":449573222333284353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573222333284353,"checkpoint_ts":449573222333284353,"checkpoint_time":"2024-05-06 16:55:41.933"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 12.52 secs (292814772 bytes/sec) [Pipeline] { [Pipeline] cache + set +x [Mon May 6 16:55:52 CST 2024] <<<<<< START kafka consumer in cli_tls_with_auth case >>>>>> table test.simple not exists for 1-th check, retry later table test.simple not exists for 2-th check, retry later table test.simple exists table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573226160586753, "checkpoint_time": "2024-05-06 16:55:56.533", "error": null }' + echo '{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573226160586753, "checkpoint_time": "2024-05-06 16:55:56.533", "error": null }' { "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573226160586753, "checkpoint_time": "2024-05-06 16:55:56.533", "error": null } ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449573226160586753, '"checkpoint_time":' '"2024-05-06' '16:55:56.533",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449573226160586753, '"checkpoint_time":' '"2024-05-06' '16:55:56.533",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0016e9bb0} {Type:update Path:[Config SyncPointRetention] From: To:0xc0016e9bb8} {Type:update Path:[Config Consistent] From: To:0xc0018a5c00} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4000.out cli changefeed --changefeed-id custom-changefeed-name pause =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:55:57 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 8.12 secs (451405998 bytes/sec) [Pipeline] { + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449573226422730754, "checkpoint_time": "2024-05-06 16:55:57.533", "error": null }' + echo '{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449573226422730754, "checkpoint_time": "2024-05-06 16:55:57.533", "error": null }' { "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449573226422730754, "checkpoint_time": "2024-05-06 16:55:57.533", "error": null } ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449573226422730754, '"checkpoint_time":' '"2024-05-06' '16:55:57.533",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449573226422730754, '"checkpoint_time":' '"2024-05-06' '16:55:57.533",' '"error":' null '}' ++ jq -r .error.message [Pipeline] } [Pipeline] container [Pipeline] { + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4086.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc003b989f8} {Type:update Path:[Config SyncPointRetention] From: To:0xc003b98a08} {Type:update Path:[Config Consistent] From: To:0xc00150a2a0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365807625120241797,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-8436?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:55:51.132577511+08:00","start_ts":449573222333284353,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":0,"checkpoint_ts":449573226422730754,"checkpoint_time":"2024-05-06 16:55:57.533"} PASS [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh start tidb cluster in /tmp/tidb_cdc_test/kafka_sink_error_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... coverage: 2.8% of statements in github.com/pingcap/tiflow/... [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] // container [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + set +x [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] } + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4123.out cli changefeed --changefeed-id custom-changefeed-name resume + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=33d8ae09-5d5f-4a73-8518-e6cabea98072 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-k4x4v GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr pingcap_tiflow_pull_cdc_integration_kafka_test_1870-k4x4v GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x [Pipeline] // timeout [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // container [Pipeline] sh start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=5956cdb3-c068-4a07-beaa-1c9d87170f01 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-j10vp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x pingcap_tiflow_pull_cdc_integration_kafka_test_1870-j10vp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] sh [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=9cb16373-f56d-4d4f-aa67-80969e3f66d4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-t2mdb GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-t2mdb pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:56:11 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573229830602755, "checkpoint_time": "2024-05-06 16:56:10.533", "error": null }' + echo '{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573229830602755, "checkpoint_time": "2024-05-06 16:56:10.533", "error": null }' { "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573229830602755, "checkpoint_time": "2024-05-06 16:56:10.533", "error": null } ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449573229830602755, '"checkpoint_time":' '"2024-05-06' '16:56:10.533",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449573229830602755, '"checkpoint_time":' '"2024-05-06' '16:56:10.533",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4212.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449573230092746755 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-8436?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... + set +x Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3478aeb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:9269, start at 2024-05-06 16:56:14.028495088 +0800 CST m=+5.087689435 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:14.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:13.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:13.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3478aeb00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:9269, start at 2024-05-06 16:56:14.028495088 +0800 CST m=+5.087689435 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:14.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:13.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:13.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3478b1d40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:9347, start at 2024-05-06 16:56:14.226353945 +0800 CST m=+5.235472172 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:14.233 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:14.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:14.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDA7A66D7911B < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 08:56:17 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 16:56:17 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT [Mon May 6 16:56:17 CST 2024] <<<<<< START cdc server in kafka_sink_error_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_sink_error_resume.1066610668.out server --log-file /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4292.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-8436?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365807625120241797,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-8436?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:56:19.19004195+08:00","start_ts":449573232032612357,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573232032612357,"checkpoint_ts":449573232032612357,"checkpoint_time":"2024-05-06 16:56:18.933"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a398912f-e03d-4219-96b9-5b3bf71d88df {"id":"a398912f-e03d-4219-96b9-5b3bf71d88df","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985777} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e05f6c8 a398912f-e03d-4219-96b9-5b3bf71d88df /tidb/cdc/default/default/upstream/7365807777552391744 {"id":7365807777552391744,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a398912f-e03d-4219-96b9-5b3bf71d88df {"id":"a398912f-e03d-4219-96b9-5b3bf71d88df","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985777} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e05f6c8 a398912f-e03d-4219-96b9-5b3bf71d88df /tidb/cdc/default/default/upstream/7365807777552391744 {"id":7365807777552391744,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a398912f-e03d-4219-96b9-5b3bf71d88df {"id":"a398912f-e03d-4219-96b9-5b3bf71d88df","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985777} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e05f6c8 a398912f-e03d-4219-96b9-5b3bf71d88df /tidb/cdc/default/default/upstream/7365807777552391744 {"id":7365807777552391744,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x [Mon May 6 16:56:20 CST 2024] <<<<<< START kafka consumer in kafka_sink_error_resume case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347924d00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr, pid:1403, start at 2024-05-06 16:56:21.57725594 +0800 CST m=+5.712912748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:21.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:21.556 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:21.556 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 571 0 --:--:-- --:--:-- --:--:-- 573 + info='{"state":"normal","resolved_ts":449573232599892001,"checkpoint_ts":449573232599892001}' + echo '{"state":"normal","resolved_ts":449573232599892001,"checkpoint_ts":449573232599892001}' {"state":"normal","resolved_ts":449573232599892001,"checkpoint_ts":449573232599892001} ++ echo '{"state":"normal","resolved_ts":449573232599892001,"checkpoint_ts":449573232599892001}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573232976330756, "checkpoint_time": "2024-05-06 16:56:22.533", "error": null }' + echo '{ "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573232976330756, "checkpoint_time": "2024-05-06 16:56:22.533", "error": null }' { "upstream_id": 7365807625120241797, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449573232976330756, "checkpoint_time": "2024-05-06 16:56:22.533", "error": null } ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449573232976330756, '"checkpoint_time":' '"2024-05-06' '16:56:22.533",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365807625120241797, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449573232976330756, '"checkpoint_time":' '"2024-05-06' '16:56:22.533",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:56:23 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347924d00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr, pid:1403, start at 2024-05-06 16:56:21.57725594 +0800 CST m=+5.712912748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:21.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:21.556 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:21.556 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34791dc00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr, pid:1486, start at 2024-05-06 16:56:21.146546585 +0800 CST m=+5.221487048 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:21.155 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:21.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:21.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 782 0 --:--:-- --:--:-- --:--:-- 788 + info='{"state":"normal","resolved_ts":449573233360109569,"checkpoint_ts":449573232743809036}' + echo '{"state":"normal","resolved_ts":449573233360109569,"checkpoint_ts":449573232743809036}' {"state":"normal","resolved_ts":449573233360109569,"checkpoint_ts":449573232743809036} ++ echo '{"state":"normal","resolved_ts":449573233360109569,"checkpoint_ts":449573232743809036}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 2-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4416.out cli changefeed create --start-ts=449573222333284353 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-11058?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: d504519f-da56-4996-a915-43703ed6c288 Info: {"upstream_id":7365807625120241797,"namespace":"default","id":"d504519f-da56-4996-a915-43703ed6c288","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-11058?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-05-06T16:56:25.738738074+08:00","start_ts":449573222333284353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573222333284353,"checkpoint_ts":449573222333284353,"checkpoint_time":"2024-05-06 16:55:41.933"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34795e700018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:1403, start at 2024-05-06 16:56:25.293008276 +0800 CST m=+5.286393825 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:25.299 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:25.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:25.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:56:26 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28752877.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4458.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34795e700018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:1403, start at 2024-05-06 16:56:25.293008276 +0800 CST m=+5.286393825 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:25.299 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:25.294 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:25.294 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347960240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:1479, start at 2024-05-06 16:56:25.391631477 +0800 CST m=+5.322881472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:58:25.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:56:25.403 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:46:25.403 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:56:28 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2193 0 --:--:-- --:--:-- --:--:-- 2209 + info='{"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .last_warning.message + error_msg='kafka sink injected error' + [[ ! kafka sink injected error =~ kafka ]] run task successfully + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4496.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 check_changefeed_status 127.0.0.1:8300 48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 normal + endpoint=127.0.0.1:8300 + changefeed_id=48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1829 0 --:--:-- --:--:-- --:--:-- 1840 + info='{"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449573234408685570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:29 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ee2caf5-5069-4262-b8f3-1b3e6088d572 {"id":"5ee2caf5-5069-4262-b8f3-1b3e6088d572","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985786} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e1b18ce 5ee2caf5-5069-4262-b8f3-1b3e6088d572 /tidb/cdc/default/default/upstream/7365807807300808798 {"id":7365807807300808798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ee2caf5-5069-4262-b8f3-1b3e6088d572 {"id":"5ee2caf5-5069-4262-b8f3-1b3e6088d572","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985786} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e1b18ce 5ee2caf5-5069-4262-b8f3-1b3e6088d572 /tidb/cdc/default/default/upstream/7365807807300808798 {"id":7365807807300808798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ee2caf5-5069-4262-b8f3-1b3e6088d572 {"id":"5ee2caf5-5069-4262-b8f3-1b3e6088d572","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985786} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e1b18ce 5ee2caf5-5069-4262-b8f3-1b3e6088d572 /tidb/cdc/default/default/upstream/7365807807300808798 {"id":7365807807300808798,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 4ed80589-3b5f-4ee1-a628-5e7056594609 Info: {"upstream_id":7365807807300808798,"namespace":"default","id":"4ed80589-3b5f-4ee1-a628-5e7056594609","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T16:56:29.592421664+08:00","start_ts":449573233952555010,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573233952555010,"checkpoint_ts":449573233952555010,"checkpoint_time":"2024-05-06 16:56:26.257"} [Mon May 6 16:56:29 CST 2024] <<<<<< START kafka consumer in charset_gbk case >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2811.out cli tso query --pd=http://127.0.0.1:2379 + set +x check_changefeed_status 127.0.0.1:8300 48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 normal + endpoint=127.0.0.1:8300 + changefeed_id=48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:56:31 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2155 0 --:--:-- --:--:-- --:--:-- 2150 100 243 100 243 0 0 2153 0 --:--:-- --:--:-- --:--:-- 2150 + info='{"state":"warning","resolved_ts":449573235195117570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449573235195117570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449573235195117570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449573235195117570,"checkpoint_ts":449573232743809036,"last_warning":{"time":"2024-05-06T16:56:24.57981582+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later + set +x + tso='449573235115950081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573235115950081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "scanproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "threadcount"="2" "recordcount"="10" "operationcount"="0" "workload"="core" "mysql.db"="sink_retry" "readproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "updateproportion"="0" "insertproportion"="0" "dotransactions"="false" "mysql.user"="root" ********************************************** Run finished, takes 9.029343ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1958.3, Avg(us): 1729, Min(us): 919, Max(us): 3869, 95th(us): 4000, 99th(us): 4000 [Mon May 6 16:56:32 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.28722874.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 19.35 secs (189344818 bytes/sec) [Pipeline] { [Pipeline] cache + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4589.out cli unsafe resolve-lock --region=88 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:56:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7338f46-03ae-4146-b980-d7cdd38ecbfb {"id":"e7338f46-03ae-4146-b980-d7cdd38ecbfb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985792} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e2b40ee e7338f46-03ae-4146-b980-d7cdd38ecbfb /tidb/cdc/default/default/upstream/7365807824486740645 {"id":7365807824486740645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7338f46-03ae-4146-b980-d7cdd38ecbfb {"id":"e7338f46-03ae-4146-b980-d7cdd38ecbfb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985792} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e2b40ee e7338f46-03ae-4146-b980-d7cdd38ecbfb /tidb/cdc/default/default/upstream/7365807824486740645 {"id":7365807824486740645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e7338f46-03ae-4146-b980-d7cdd38ecbfb {"id":"e7338f46-03ae-4146-b980-d7cdd38ecbfb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985792} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1e2b40ee e7338f46-03ae-4146-b980-d7cdd38ecbfb /tidb/cdc/default/default/upstream/7365807824486740645 {"id":7365807824486740645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2926.out cli changefeed create --start-ts=449573235115950081 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-4382?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' check_changefeed_status 127.0.0.1:8300 48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 normal + endpoint=127.0.0.1:8300 + changefeed_id=48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/48fa6da6-4288-4c2f-b33b-dbcdeff2c5e8/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 773 0 --:--:-- --:--:-- --:--:-- 774 + info='{"state":"normal","resolved_ts":449573236243431426,"checkpoint_ts":449573236243431426}' + echo '{"state":"normal","resolved_ts":449573236243431426,"checkpoint_ts":449573236243431426}' {"state":"normal","resolved_ts":449573236243431426,"checkpoint_ts":449573236243431426} ++ echo '{"state":"normal","resolved_ts":449573236243431426,"checkpoint_ts":449573236243431426}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z '' ]] ++ echo '{"state":"normal","resolved_ts":449573236243431426,"checkpoint_ts":449573236243431426}' ++ jq -r .last_error + error_msg=null + [[ ! null == \n\u\l\l ]] ++ echo '{"state":"normal","resolved_ts":449573236243431426,"checkpoint_ts":449573236243431426}' ++ jq -r .last_warning + error_msg=null + [[ ! null == \n\u\l\l ]] + exit 0 run task successfully table kafka_sink_error_resume.t1 exists table kafka_sink_error_resume.t2 exists check diff successfully Create changefeed successfully! ID: 2ff99715-a409-45d1-9066-89d86cd7eead Info: {"upstream_id":7365807824486740645,"namespace":"default","id":"2ff99715-a409-45d1-9066-89d86cd7eead","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-4382?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:56:35.940551924+08:00","start_ts":449573235115950081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573235115950081,"checkpoint_ts":449573235115950081,"checkpoint_time":"2024-05-06 16:56:30.695"} PASS check diff failed 1-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4622.out cli unsafe resolve-lock --region=88 --ts=449573235230769157 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 16:56:37 CST 2024] <<<<<< START kafka consumer in sink_retry case >>>>>> table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1633 46 --:--:-- --:--:-- --:--:-- 1643 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" check diff successfully wait process cdc.test exit for 1-th time... table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:56:39 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide table test.finish_mark not exists for 2-th check, retry later * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDA7FE31F09CA < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 08:56:40 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 16:56:41 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 181 100 181 0 0 1363 0 --:--:-- --:--:-- --:--:-- 1371 { "version": "v8.2.0-alpha-22-ge9f061d7a", "git_hash": "e9f061d7a0898b066e3c6e271294672cae36f6d6", "id": "7f462852-7fbc-4145-a095-48870ccca378", "pid": 3689, "is_owner": true }wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 7.88 secs (464802072 bytes/sec) [Pipeline] { [Pipeline] cache cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:56:43 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> table test.finish_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:56:44 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 4-th check, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:56:48 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: kafka... <<================= [Mon May 6 16:56:51 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 8.32 secs (440647203 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/mq_sink_dispatcher Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347be03c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:2172, start at 2024-05-06 16:57:06.330643493 +0800 CST m=+5.130542028 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:06.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:06.319 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:06.319 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347be03c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:2172, start at 2024-05-06 16:57:06.330643493 +0800 CST m=+5.130542028 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:06.338 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:06.319 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:06.319 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347be1d00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:2252, start at 2024-05-06 16:57:06.465246497 +0800 CST m=+5.206182473 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:06.472 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:06.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:06.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 9.71 secs (377503213 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347bfc80000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:12261, start at 2024-05-06 16:57:08.140538079 +0800 CST m=+5.519557687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:08.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:08.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:08.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="sink_retry" "operationcount"="0" "mysql.port"="4000" "insertproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "threadcount"="2" "workload"="core" "recordcount"="10" "scanproportion"="0" "readallfields"="true" "updateproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 5.183381ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2769.1, Avg(us): 925, Min(us): 621, Max(us): 2209, 95th(us): 3000, 99th(us): 3000 [Mon May 6 16:57:09 CST 2024] <<<<<< START cdc server in kafka_big_messages_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages_v2.36273629.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347c1ce80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:5769, start at 2024-05-06 16:57:10.246397319 +0800 CST m=+5.598748429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:10.253 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:10.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:10.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347bfc80000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:12261, start at 2024-05-06 16:57:08.140538079 +0800 CST m=+5.519557687 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:08.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:08.128 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:08.128 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347bfd340016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:12347, start at 2024-05-06 16:57:08.218852179 +0800 CST m=+5.543989706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:08.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:08.224 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:08.224 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:12 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a98666e5-0336-4c90-a585-39d79b8b0557 {"id":"a98666e5-0336-4c90-a585-39d79b8b0557","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985829} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ecc17ca a98666e5-0336-4c90-a585-39d79b8b0557 /tidb/cdc/default/default/upstream/7365807990195410287 {"id":7365807990195410287,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a98666e5-0336-4c90-a585-39d79b8b0557 {"id":"a98666e5-0336-4c90-a585-39d79b8b0557","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985829} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ecc17ca a98666e5-0336-4c90-a585-39d79b8b0557 /tidb/cdc/default/default/upstream/7365807990195410287 {"id":7365807990195410287,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a98666e5-0336-4c90-a585-39d79b8b0557 {"id":"a98666e5-0336-4c90-a585-39d79b8b0557","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985829} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ecc17ca a98666e5-0336-4c90-a585-39d79b8b0557 /tidb/cdc/default/default/upstream/7365807990195410287 {"id":7365807990195410287,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 2f004dce-4c43-453a-b93d-7c83c004bc2b Info: {"upstream_id":7365807990195410287,"namespace":"default","id":"2f004dce-4c43-453a-b93d-7c83c004bc2b","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-06T16:57:12.913156701+08:00","start_ts":449573245306273793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573245306273793,"checkpoint_ts":449573245306273793,"checkpoint_time":"2024-05-06 16:57:09.568"} [Mon May 6 16:57:12 CST 2024] <<<<<< START cdc server in mq_sink_dispatcher case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.1367913681.out server --log-file /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc.log --log-level info --data-dir /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 16:57:12 CST 2024] <<<<<< START kafka consumer in kafka_big_messages_v2 case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347c1ce80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:5769, start at 2024-05-06 16:57:10.246397319 +0800 CST m=+5.598748429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:10.253 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:10.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:10.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347c1f400016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:5855, start at 2024-05-06 16:57:10.399276509 +0800 CST m=+5.692539388 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:10.406 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:10.403 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:10.403 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:57:15 CST 2024] <<<<<< START cdc server in kafka_simple_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.71867188.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:16 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d753e172-eba9-49d9-8e2a-f94ed5c5bdba {"id":"d753e172-eba9-49d9-8e2a-f94ed5c5bdba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985833} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ed5b4ce d753e172-eba9-49d9-8e2a-f94ed5c5bdba /tidb/cdc/default/default/upstream/7365808009265599262 {"id":7365808009265599262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d753e172-eba9-49d9-8e2a-f94ed5c5bdba {"id":"d753e172-eba9-49d9-8e2a-f94ed5c5bdba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985833} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ed5b4ce d753e172-eba9-49d9-8e2a-f94ed5c5bdba /tidb/cdc/default/default/upstream/7365808009265599262 {"id":7365808009265599262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d753e172-eba9-49d9-8e2a-f94ed5c5bdba {"id":"d753e172-eba9-49d9-8e2a-f94ed5c5bdba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985833} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ed5b4ce d753e172-eba9-49d9-8e2a-f94ed5c5bdba /tidb/cdc/default/default/upstream/7365808009265599262 {"id":7365808009265599262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.13753.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347c6c280009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr, pid:4164, start at 2024-05-06 16:57:15.286302985 +0800 CST m=+5.873614207 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:15.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:15.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:15.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347c6c280009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr, pid:4164, start at 2024-05-06 16:57:15.286302985 +0800 CST m=+5.873614207 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:15.294 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:15.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:15.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347c63780003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-k4x4v-hkmjr, pid:4240, start at 2024-05-06 16:57:14.718923888 +0800 CST m=+5.253760520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:14.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:14.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:14.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449573247143641093 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573247143641093 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.13786.out cli changefeed create --start-ts=449573247143641093 '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/changefeed.toml table kafka_big_messages.test not exists for 1-th check, retry later Create changefeed successfully! ID: test Info: {"upstream_id":7365808009265599262,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:57:18.52246942+08:00","start_ts":449573247143641093,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["verify.t"],"partition":"index-value"},{"matcher":["dispatcher.index"],"partition":"index-value","index":"idx_a"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573247143641093,"checkpoint_ts":449573247143641093,"checkpoint_time":"2024-05-06 16:57:16.577"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b750f959-7052-4589-b3d7-403b56cdf181 {"id":"b750f959-7052-4589-b3d7-403b56cdf181","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985835} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ed981d3 b750f959-7052-4589-b3d7-403b56cdf181 /tidb/cdc/default/default/upstream/7365808006410687775 {"id":7365808006410687775,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b750f959-7052-4589-b3d7-403b56cdf181 {"id":"b750f959-7052-4589-b3d7-403b56cdf181","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985835} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ed981d3 b750f959-7052-4589-b3d7-403b56cdf181 /tidb/cdc/default/default/upstream/7365808006410687775 {"id":7365808006410687775,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b750f959-7052-4589-b3d7-403b56cdf181 {"id":"b750f959-7052-4589-b3d7-403b56cdf181","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985835} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1ed981d3 b750f959-7052-4589-b3d7-403b56cdf181 /tidb/cdc/default/default/upstream/7365808006410687775 {"id":7365808006410687775,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.7247.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-19935?protocol=simple' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/conf/changefeed.toml -c simple-basic Create changefeed successfully! ID: simple-basic Info: {"upstream_id":7365808006410687775,"namespace":"default","id":"simple-basic","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-19935?protocol=simple","create_time":"2024-05-06T16:57:19.138723019+08:00","start_ts":449573247779340292,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":5,"send_bootstrap_in_msg_count":100,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573247779340292,"checkpoint_ts":449573247779340292,"checkpoint_time":"2024-05-06 16:57:19.002"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Mon May 6 16:57:19 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.56405642.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573247143641093, "checkpoint_time": "2024-05-06 16:57:16.577", "error": null }' + echo '{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573247143641093, "checkpoint_time": "2024-05-06 16:57:16.577", "error": null }' { "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573247143641093, "checkpoint_time": "2024-05-06 16:57:16.577", "error": null } ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449573247143641093, '"checkpoint_time":' '"2024-05-06' '16:57:16.577",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449573247143641093, '"checkpoint_time":' '"2024-05-06' '16:57:16.577",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + set +x check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573247143641093, "checkpoint_time": "2024-05-06 16:57:16.577", "error": null }' + echo '{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573247143641093, "checkpoint_time": "2024-05-06 16:57:16.577", "error": null }' { "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573247143641093, "checkpoint_time": "2024-05-06 16:57:16.577", "error": null } ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449573247143641093, '"checkpoint_time":' '"2024-05-06' '16:57:16.577",' '"error":' null '}' ++ jq -r .state table kafka_big_messages.test exists check diff failed 1-th time, retry later + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:22 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0cd9ea0-a0b6-444e-8ea5-61788dc84733 {"id":"e0cd9ea0-a0b6-444e-8ea5-61788dc84733","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985839} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec26cf e0cd9ea0-a0b6-444e-8ea5-61788dc84733 /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0cd9ea0-a0b6-444e-8ea5-61788dc84733 {"id":"e0cd9ea0-a0b6-444e-8ea5-61788dc84733","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985839} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec26cf e0cd9ea0-a0b6-444e-8ea5-61788dc84733 /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0cd9ea0-a0b6-444e-8ea5-61788dc84733 {"id":"e0cd9ea0-a0b6-444e-8ea5-61788dc84733","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985839} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec26cf e0cd9ea0-a0b6-444e-8ea5-61788dc84733 /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5696.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-manager check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365808025716056435,"namespace":"default","id":"ddl-manager","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:57:22.89493909+08:00","start_ts":449573248768147460,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573248768147460,"checkpoint_ts":449573248768147460,"checkpoint_time":"2024-05-06 16:57:22.774"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + info='{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": null }' + echo '{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": null }' { "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": null } ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449573248140050460, '"checkpoint_time":' '"2024-05-06' '16:57:20.378",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 2-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + set +x [Mon May 6 16:57:24 CST 2024] <<<<<< START kafka consumer in ddl_manager case >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:57:24 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> [Mon May 6 16:57:25 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic case >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870/tiflow-cdc) 3664751616 bytes in 13.85 secs (264543572 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] // container table test.finish_mark_for_ddl not exists for 1-th check, retry later [Pipeline] // container [Pipeline] // container [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=11db57ed-5498-40ad-a979-63b44595dc74 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-cq1g3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-cq1g3 pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": { "time": "2024-05-06T16:57:23.937852023+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' + echo '{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": { "time": "2024-05-06T16:57:23.937852023+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' { "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": { "time": "2024-05-06T16:57:23.937852023+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } } ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449573248140050460, '"checkpoint_time":' '"2024-05-06' '16:57:20.378",' '"error":' '{' '"time":' '"2024-05-06T16:57:23.937852023+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449573248140050460, '"checkpoint_time":' '"2024-05-06' '16:57:20.378",' '"error":' '{' '"time":' '"2024-05-06T16:57:23.937852023+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=4b5c0ae7-4cc5-4dba-b56f-333bc0685a9f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-x9nnr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-x9nnr pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { + message='[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a' + [[ ! [CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a =~ ErrDispatcherFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14029.out cli changefeed update -c test '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/new_changefeed.toml --no-confirm [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] // container [Pipeline] sh Diff of changefeed config: {Type:update Path:[Config SyncPointInterval] From: To:0xc0039227a8} {Type:update Path:[Config SyncPointRetention] From: To:0xc0039227b8} {Type:update Path:[Config Sink DispatchRules 0 Matcher 0] From:verify.t To:dispatcher.index} {Type:delete Path:[Config Sink DispatchRules 1 Matcher 0] From:dispatcher.index To:} {Type:delete Path:[Config Sink DispatchRules 1 PartitionRule] From:index-value To:} {Type:delete Path:[Config Sink DispatchRules 1 IndexName] From:idx_a To:} {Type:update Path:[Config Consistent] From: To:0xc001364700} Update changefeed config successfully! ID: test Info: {"upstream_id":7365808009265599262,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:57:18.52246942+08:00","start_ts":449573247143641093,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","dispatchers":[{"matcher":["dispatcher.index"],"partition":"index-value"}],"encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"failed","error":{"addr":"127.0.0.1:8300","code":"CDC:ErrDispatcherFailed","message":"[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a"},"creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":0,"checkpoint_ts":449573248140050460,"checkpoint_time":"2024-05-06 16:57:20.378"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=edc32b53-9245-48df-a551-ecb8b55e6944 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-0v21p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4 pingcap_tiflow_pull_cdc_integration_kafka_test_1870-0v21p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 table test.finish_mark_for_ddl not exists for 2-th check, retry later [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14077.out cli changefeed resume -c test + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + + echo dump nc localhost 2181 + awk {$1=$1;print} + grep brokers + grep -F -w /brokers/ids/1 /brokers/ids/1 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table test.finish_mark_for_ddl not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": null }' + echo '{ "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": null }' { "upstream_id": 7365808009265599262, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449573248140050460, "checkpoint_time": "2024-05-06 16:57:20.378", "error": null } start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449573248140050460, '"checkpoint_time":' '"2024-05-06' '16:57:20.378",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808009265599262, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449573248140050460, '"checkpoint_time":' '"2024-05-06' '16:57:20.378",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=abadbe84-83c5-495c-adbe-8c8b1dcecba7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dg389 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6 pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dg389 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.finish_mark_for_ddl not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark_for_ddl not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Mon May 6 16:57:36 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout table test.finish_mark_for_ddl exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.7365.out cli changefeed pause -c simple-basic [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] // container ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] // container [Pipeline] sh [Pipeline] sh Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.7401.out cli changefeed resume -c simple-basic [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=66093ca5-8886-4555-87dd-e54b569afe89 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4z7q7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9 pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4z7q7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: kafka... <<================= [Pipeline] sh =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=db9c57cd-11c3-470e-97a3-5e048481a260 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4tqk4 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-4tqk4 pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G10 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=e5cfbaca-e63d-4226-9c74-699933c5555d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-mvw62 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb pingcap_tiflow_pull_cdc_integration_kafka_test_1870-mvw62 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: kafka... <<================= Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=628a9d4d-2348-436e-b005-068f15d2715c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dqtbf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm pingcap_tiflow_pull_cdc_integration_kafka_test_1870-dqtbf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: kafka... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad JENKINS_NODE_COOKIE=ba71b123-4b02-4909-af8d-831af6ca48ca BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1870 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787402136024256513","prowjobid":"9ad3c2b0-d1b6-494f-8ab8-d7e0efa01aad","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":11023,"author":"CharlesCheung96","sha":"e9f061d7a0898b066e3c6e271294672cae36f6d6","title":"test(ticdc): enable sequence test","link":"https://github.com/pingcap/tiflow/pull/11023","commit_link":"https://github.com/pingcap/tiflow/pull/11023/commits/e9f061d7a0898b066e3c6e271294672cae36f6d6","author_link":"https://github.com/CharlesCheung96"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1870 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787402136024256513 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1870/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1870-2v9k0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb pingcap_tiflow_pull_cdc_integration_kafka_test_1870-2v9k0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1870 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:57:39 CST 2024] <<<<<< run test case sink_retry success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347df9000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:1376, start at 2024-05-06 16:57:40.70231778 +0800 CST m=+5.090620695 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:40.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:40.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:40.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347df9000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:1376, start at 2024-05-06 16:57:40.70231778 +0800 CST m=+5.090620695 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:40.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:40.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:40.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347dfc100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:1438, start at 2024-05-06 16:57:40.905974033 +0800 CST m=+5.244998151 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:40.916 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:40.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:40.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process 5645 exit for 1-th time... wait process 5645 exit for 2-th time... wait process 5645 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5645) - No such process wait process 5645 exit for 4-th time... process 5645 already exit [Mon May 6 16:57:41 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.58095811.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e07c40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g, pid:1339, start at 2024-05-06 16:57:41.642331786 +0800 CST m=+5.469906417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:41.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:41.617 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:41.617 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2749.out cli tso query --pd=http://127.0.0.1:2379 The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 1 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e21f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:1347, start at 2024-05-06 16:57:43.33330483 +0800 CST m=+5.296992430 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:43.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:43.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:43.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e07c40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g, pid:1339, start at 2024-05-06 16:57:41.642331786 +0800 CST m=+5.469906417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:41.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:41.617 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:41.617 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e0958000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g, pid:1420, start at 2024-05-06 16:57:41.734705847 +0800 CST m=+5.503373496 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:41.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:41.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:41.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449573254403719169 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573254403719169 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:57:45 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.27782780.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e21f00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:1347, start at 2024-05-06 16:57:43.33330483 +0800 CST m=+5.296992430 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:43.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:43.342 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:43.342 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e23600013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:1432, start at 2024-05-06 16:57:43.411238444 +0800 CST m=+5.321044000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:43.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:43.384 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:43.384 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 16:57:46 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.27952797.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365808025716056435 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:57:22.89493909 +0800 CST StartTs:449573248768147460 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002636240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573248794361859} {CheckpointTs:449573249449721880 MinTableBarrierTs:449573249449721880 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98f5ec1-968a-4d29-ad80-010b025bf378 {"id":"c98f5ec1-968a-4d29-ad80-010b025bf378","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec29bf c98f5ec1-968a-4d29-ad80-010b025bf378 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365808025716056435,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:57:22.89493909+08:00","start-ts":449573248768147460,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573248794361859} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573249449721880,"min-table-barrier-ts":449573249449721880,"admin-job-type":0} /tidb/cdc/default/default/task/position/c98f5ec1-968a-4d29-ad80-010b025bf378/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365808025716056435 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:57:22.89493909 +0800 CST StartTs:449573248768147460 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002636240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573248794361859} {CheckpointTs:449573249449721880 MinTableBarrierTs:449573249449721880 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98f5ec1-968a-4d29-ad80-010b025bf378 {"id":"c98f5ec1-968a-4d29-ad80-010b025bf378","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec29bf c98f5ec1-968a-4d29-ad80-010b025bf378 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365808025716056435,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:57:22.89493909+08:00","start-ts":449573248768147460,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573248794361859} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573249449721880,"min-table-barrier-ts":449573249449721880,"admin-job-type":0} /tidb/cdc/default/default/task/position/c98f5ec1-968a-4d29-ad80-010b025bf378/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365808025716056435 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:57:22.89493909 +0800 CST StartTs:449573248768147460 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002636240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573248794361859} {CheckpointTs:449573249449721880 MinTableBarrierTs:449573249449721880 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98f5ec1-968a-4d29-ad80-010b025bf378 {"id":"c98f5ec1-968a-4d29-ad80-010b025bf378","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec29bf c98f5ec1-968a-4d29-ad80-010b025bf378 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365808025716056435,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:57:22.89493909+08:00","start-ts":449573248768147460,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573248794361859} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573249449721880,"min-table-barrier-ts":449573249449721880,"admin-job-type":0} /tidb/cdc/default/default/task/position/c98f5ec1-968a-4d29-ad80-010b025bf378/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 16:57:47 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.58785880.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table test.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh: line 1: 14160 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/new_changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:48 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e91496ed-4e0c-480f-858b-c234bf39aa35 {"id":"e91496ed-4e0c-480f-858b-c234bf39aa35","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f5695d1 e91496ed-4e0c-480f-858b-c234bf39aa35 /tidb/cdc/default/default/upstream/7365808151209793441 {"id":7365808151209793441,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e91496ed-4e0c-480f-858b-c234bf39aa35 {"id":"e91496ed-4e0c-480f-858b-c234bf39aa35","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f5695d1 e91496ed-4e0c-480f-858b-c234bf39aa35 /tidb/cdc/default/default/upstream/7365808151209793441 {"id":7365808151209793441,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e91496ed-4e0c-480f-858b-c234bf39aa35 {"id":"e91496ed-4e0c-480f-858b-c234bf39aa35","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f5695d1 e91496ed-4e0c-480f-858b-c234bf39aa35 /tidb/cdc/default/default/upstream/7365808151209793441 {"id":7365808151209793441,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2828.out cli changefeed create --start-ts=449573254403719169 '--sink-uri=kafka://127.0.0.1:9092/ticdc-row-format-test-16322?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 79d6316f-7f5c-4bf7-83dd-417b17516893 Info: {"upstream_id":7365808151209793441,"namespace":"default","id":"79d6316f-7f5c-4bf7-83dd-417b17516893","sink_uri":"kafka://127.0.0.1:9092/ticdc-row-format-test-16322?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:57:49.275746185+08:00","start_ts":449573254403719169,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573254403719169,"checkpoint_ts":449573254403719169,"checkpoint_time":"2024-05-06 16:57:44.272"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e6c000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6, pid:1315, start at 2024-05-06 16:57:48.061378163 +0800 CST m=+7.290070405 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:48.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:48.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:48.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e6c000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6, pid:1315, start at 2024-05-06 16:57:48.061378163 +0800 CST m=+7.290070405 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:48.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:48.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:48.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e6d300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6, pid:1360, start at 2024-05-06 16:57:48.133643549 +0800 CST m=+7.251225256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:48.156 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:48.158 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:48.158 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22175e6b-0f72-430a-bd6f-3615e5cc8b53 {"id":"22175e6b-0f72-430a-bd6f-3615e5cc8b53","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f58b5cc 22175e6b-0f72-430a-bd6f-3615e5cc8b53 /tidb/cdc/default/default/upstream/7365808151809051565 {"id":7365808151809051565,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22175e6b-0f72-430a-bd6f-3615e5cc8b53 {"id":"22175e6b-0f72-430a-bd6f-3615e5cc8b53","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f58b5cc 22175e6b-0f72-430a-bd6f-3615e5cc8b53 /tidb/cdc/default/default/upstream/7365808151809051565 {"id":7365808151809051565,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22175e6b-0f72-430a-bd6f-3615e5cc8b53 {"id":"22175e6b-0f72-430a-bd6f-3615e5cc8b53","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f58b5cc 22175e6b-0f72-430a-bd6f-3615e5cc8b53 /tidb/cdc/default/default/upstream/7365808151809051565 {"id":7365808151809051565,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Mon May 6 16:57:49 CST 2024] <<<<<< START kafka consumer in kv_client_stream_reconnect case >>>>>> table test.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Mon May 6 16:57:50 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.29352937.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [Mon May 6 16:57:50 CST 2024] <<<<<< START kafka consumer in row_format case >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2682.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e9d280010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:5889, start at 2024-05-06 16:57:51.193409532 +0800 CST m=+5.178413446 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:51.201 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:51.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:51.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365808025716056435 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:57:22.89493909 +0800 CST StartTs:449573248768147460 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002636240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573248794361859} {CheckpointTs:449573249449721880 MinTableBarrierTs:449573249449721880 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98f5ec1-968a-4d29-ad80-010b025bf378 {"id":"c98f5ec1-968a-4d29-ad80-010b025bf378","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec29bf c98f5ec1-968a-4d29-ad80-010b025bf378 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365808025716056435,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:57:22.89493909+08:00","start-ts":449573248768147460,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573248794361859} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573249449721880,"min-table-barrier-ts":449573249449721880,"admin-job-type":0} /tidb/cdc/default/default/task/position/c98f5ec1-968a-4d29-ad80-010b025bf378/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365808025716056435 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:57:22.89493909 +0800 CST StartTs:449573248768147460 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002636240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573248794361859} {CheckpointTs:449573249449721880 MinTableBarrierTs:449573249449721880 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98f5ec1-968a-4d29-ad80-010b025bf378 {"id":"c98f5ec1-968a-4d29-ad80-010b025bf378","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec29bf c98f5ec1-968a-4d29-ad80-010b025bf378 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365808025716056435,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:57:22.89493909+08:00","start-ts":449573248768147460,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573248794361859} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573249449721880,"min-table-barrier-ts":449573249449721880,"admin-job-type":0} /tidb/cdc/default/default/task/position/c98f5ec1-968a-4d29-ad80-010b025bf378/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808025716056435 + grep -q 'failed to get info:' {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365808025716056435 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:57:22.89493909 +0800 CST StartTs:449573248768147460 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002636240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573248794361859} {CheckpointTs:449573249449721880 MinTableBarrierTs:449573249449721880 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c98f5ec1-968a-4d29-ad80-010b025bf378 {"id":"c98f5ec1-968a-4d29-ad80-010b025bf378","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985864} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1eec29bf c98f5ec1-968a-4d29-ad80-010b025bf378 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365808025716056435,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-4046?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:57:22.89493909+08:00","start-ts":449573248768147460,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573248794361859} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449573249449721880,"min-table-barrier-ts":449573249449721880,"admin-job-type":0} /tidb/cdc/default/default/task/position/c98f5ec1-968a-4d29-ad80-010b025bf378/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808025716056435 {"id":7365808025716056435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449573256411480065 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573256411480065 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:57:53 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.27012703.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347ebb380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw, pid:1401, start at 2024-05-06 16:57:53.140525746 +0800 CST m=+5.346034194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:53.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:53.151 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:53.151 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:53 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8f19090-0541-4f6e-bb8b-d0153eed3836 {"id":"f8f19090-0541-4f6e-bb8b-d0153eed3836","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f5c2163 f8f19090-0541-4f6e-bb8b-d0153eed3836 /tidb/cdc/default/default/upstream/7365808156466569673 {"id":7365808156466569673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8f19090-0541-4f6e-bb8b-d0153eed3836 {"id":"f8f19090-0541-4f6e-bb8b-d0153eed3836","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f5c2163 f8f19090-0541-4f6e-bb8b-d0153eed3836 /tidb/cdc/default/default/upstream/7365808156466569673 {"id":7365808156466569673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8f19090-0541-4f6e-bb8b-d0153eed3836 {"id":"f8f19090-0541-4f6e-bb8b-d0153eed3836","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f5c2163 f8f19090-0541-4f6e-bb8b-d0153eed3836 /tidb/cdc/default/default/upstream/7365808156466569673 {"id":7365808156466569673,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e9d280010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:5889, start at 2024-05-06 16:57:51.193409532 +0800 CST m=+5.178413446 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:51.201 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:51.178 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:51.178 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347e9ea80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:5972, start at 2024-05-06 16:57:51.279108044 +0800 CST m=+5.210414218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:51.289 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:51.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:51.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 16:57:54 CST 2024] <<<<<< START kafka consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 c9d7314b-ef05-4262-9fbf-a9eda1efc592 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=c9d7314b-ef05-4262-9fbf-a9eda1efc592 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c9d7314b-ef05-4262-9fbf-a9eda1efc592 -s + info='{ "upstream_id": 7365808156466569673, "namespace": "default", "id": "c9d7314b-ef05-4262-9fbf-a9eda1efc592", "state": "normal", "checkpoint_tso": 449573256925544452, "checkpoint_time": "2024-05-06 16:57:53.892", "error": null }' + echo '{ "upstream_id": 7365808156466569673, "namespace": "default", "id": "c9d7314b-ef05-4262-9fbf-a9eda1efc592", "state": "normal", "checkpoint_tso": 449573256925544452, "checkpoint_time": "2024-05-06 16:57:53.892", "error": null }' { "upstream_id": 7365808156466569673, "namespace": "default", "id": "c9d7314b-ef05-4262-9fbf-a9eda1efc592", "state": "normal", "checkpoint_tso": 449573256925544452, "checkpoint_time": "2024-05-06 16:57:53.892", "error": null } ++ echo '{' '"upstream_id":' 7365808156466569673, '"namespace":' '"default",' '"id":' '"c9d7314b-ef05-4262-9fbf-a9eda1efc592",' '"state":' '"normal",' '"checkpoint_tso":' 449573256925544452, '"checkpoint_time":' '"2024-05-06' '16:57:53.892",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808156466569673, '"namespace":' '"default",' '"id":' '"c9d7314b-ef05-4262-9fbf-a9eda1efc592",' '"state":' '"normal",' '"checkpoint_tso":' 449573256925544452, '"checkpoint_time":' '"2024-05-06' '16:57:53.892",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347ebb380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw, pid:1401, start at 2024-05-06 16:57:53.140525746 +0800 CST m=+5.346034194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:53.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:53.151 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:53.151 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347ebcd40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw, pid:1487, start at 2024-05-06 16:57:53.24425994 +0800 CST m=+5.393410322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:53.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:53.255 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:53.255 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 16:57:56 CST 2024] <<<<<< START cdc server in multi_tables_ddl_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl_v2.74757477.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:56 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e07ab514-466a-4a3c-8f9a-5b46b578af7e {"id":"e07ab514-466a-4a3c-8f9a-5b46b578af7e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f6c6ed5 e07ab514-466a-4a3c-8f9a-5b46b578af7e /tidb/cdc/default/default/upstream/7365808172326579485 {"id":7365808172326579485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e07ab514-466a-4a3c-8f9a-5b46b578af7e {"id":"e07ab514-466a-4a3c-8f9a-5b46b578af7e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f6c6ed5 e07ab514-466a-4a3c-8f9a-5b46b578af7e /tidb/cdc/default/default/upstream/7365808172326579485 {"id":7365808172326579485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e07ab514-466a-4a3c-8f9a-5b46b578af7e {"id":"e07ab514-466a-4a3c-8f9a-5b46b578af7e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f6c6ed5 e07ab514-466a-4a3c-8f9a-5b46b578af7e /tidb/cdc/default/default/upstream/7365808172326579485 {"id":7365808172326579485,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2747.out cli changefeed create --start-ts=449573256411480065 '--sink-uri=kafka://127.0.0.1:9092/ticdc-foreign-key-test-27205?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: cb53b03d-1383-4d5d-859c-f489f9d356bd Info: {"upstream_id":7365808172326579485,"namespace":"default","id":"cb53b03d-1383-4d5d-859c-f489f9d356bd","sink_uri":"kafka://127.0.0.1:9092/ticdc-foreign-key-test-27205?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:57:57.622191153+08:00","start_ts":449573256411480065,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573256411480065,"checkpoint_ts":449573256411480065,"checkpoint_time":"2024-05-06 16:57:51.931"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347efdf80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb, pid:1358, start at 2024-05-06 16:57:57.40275197 +0800 CST m=+5.336865457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:57.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:57.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:57.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2837.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f10fc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9, pid:1473, start at 2024-05-06 16:57:58.60285468 +0800 CST m=+5.513314314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:58.610 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:58.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:58.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:57:59 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3fe190c6-d41a-409f-ad27-61f820924ad1 {"id":"3fe190c6-d41a-409f-ad27-61f820924ad1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985876} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f7f3acb 3fe190c6-d41a-409f-ad27-61f820924ad1 /tidb/cdc/default/default/upstream/7365808193414000926 {"id":7365808193414000926,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3fe190c6-d41a-409f-ad27-61f820924ad1 {"id":"3fe190c6-d41a-409f-ad27-61f820924ad1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985876} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f7f3acb 3fe190c6-d41a-409f-ad27-61f820924ad1 /tidb/cdc/default/default/upstream/7365808193414000926 {"id":7365808193414000926,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3fe190c6-d41a-409f-ad27-61f820924ad1 {"id":"3fe190c6-d41a-409f-ad27-61f820924ad1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985876} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f7f3acb 3fe190c6-d41a-409f-ad27-61f820924ad1 /tidb/cdc/default/default/upstream/7365808193414000926 {"id":7365808193414000926,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 3-th time, retry later + set +x [Mon May 6 16:57:59 CST 2024] <<<<<< START kafka consumer in foreign_key case >>>>>> Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365808193414000926,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-32409?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:57:59.727548423+08:00","start_ts":449573257577234434,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573257577234434,"checkpoint_ts":449573257577234434,"checkpoint_time":"2024-05-06 16:57:56.378"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f1a100002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb, pid:1338, start at 2024-05-06 16:57:59.204004162 +0800 CST m=+7.804313741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:59.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:59.222 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:59.222 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449573258120658946 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573258120658946 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:57:59 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.28672869.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347efdf80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb, pid:1358, start at 2024-05-06 16:57:57.40275197 +0800 CST m=+5.336865457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:57.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:57.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:57.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347eff900006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb, pid:1439, start at 2024-05-06 16:57:57.481900901 +0800 CST m=+5.356262899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:57.488 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:57.476 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:57.476 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365808193414000926,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-31496?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:01.130656416+08:00","start_ts":449573257577234434,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573257577234434,"checkpoint_ts":449573257577234434,"checkpoint_time":"2024-05-06 16:57:56.378"} table test.finish_mark not exists for 8-th check, retry later Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365808193414000926,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-31899?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:01.324814626+08:00","start_ts":449573257577234434,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573257577234434,"checkpoint_ts":449573257577234434,"checkpoint_time":"2024-05-06 16:57:56.378"} [Mon May 6 16:58:01 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Mon May 6 16:58:01 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Mon May 6 16:58:01 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f10fc0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9, pid:1473, start at 2024-05-06 16:57:58.60285468 +0800 CST m=+5.513314314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:58.610 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:58.591 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:58.591 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f11000006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4z7q7-hpqk9, pid:1556, start at 2024-05-06 16:57:58.598463045 +0800 CST m=+5.455198481 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:58.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:58.592 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:58.592 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f3b3c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:15375, start at 2024-05-06 16:58:01.32364594 +0800 CST m=+5.822421958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:01.345 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:01.345 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:01.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f1a100002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb, pid:1338, start at 2024-05-06 16:57:59.204004162 +0800 CST m=+7.804313741 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:59.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:59.222 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:59.222 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f170c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb, pid:1386, start at 2024-05-06 16:57:59.010847429 +0800 CST m=+7.523943122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:59.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:59.180 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:59.180 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 6-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2880.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f24fc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm, pid:1482, start at 2024-05-06 16:57:59.882343936 +0800 CST m=+5.415240710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:59.889 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:59.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:59.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f24fc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm, pid:1482, start at 2024-05-06 16:57:59.882343936 +0800 CST m=+5.415240710 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:59.889 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:59.871 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:59.871 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f2670000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dqtbf-lxmzm, pid:1557, start at 2024-05-06 16:57:59.977589316 +0800 CST m=+5.455250681 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-16:59:59.985 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:57:59.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:47:59.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:58:03 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> table test.finish_mark not exists for 9-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2876.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:03 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb43cc00-fe05-4ff4-b455-1e8e73d15d09 {"id":"eb43cc00-fe05-4ff4-b455-1e8e73d15d09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985880} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f881cd6 eb43cc00-fe05-4ff4-b455-1e8e73d15d09 /tidb/cdc/default/default/upstream/7365808200246925669 {"id":7365808200246925669,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb43cc00-fe05-4ff4-b455-1e8e73d15d09 {"id":"eb43cc00-fe05-4ff4-b455-1e8e73d15d09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985880} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f881cd6 eb43cc00-fe05-4ff4-b455-1e8e73d15d09 /tidb/cdc/default/default/upstream/7365808200246925669 {"id":7365808200246925669,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb43cc00-fe05-4ff4-b455-1e8e73d15d09 {"id":"eb43cc00-fe05-4ff4-b455-1e8e73d15d09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985880} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f881cd6 eb43cc00-fe05-4ff4-b455-1e8e73d15d09 /tidb/cdc/default/default/upstream/7365808200246925669 {"id":7365808200246925669,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: c467d6c1-da36-470f-b131-9235f1de440d Info: {"upstream_id":7365808200246925669,"namespace":"default","id":"c467d6c1-da36-470f-b131-9235f1de440d","sink_uri":"kafka://127.0.0.1:9092/ticdc-tiflash-test-5268?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:03.289875332+08:00","start_ts":449573258120658946,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573258120658946,"checkpoint_ts":449573258120658946,"checkpoint_time":"2024-05-06 16:57:58.451"} [Mon May 6 16:58:03 CST 2024] <<<<<< START kafka consumer in tiflash case >>>>>> table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f3b3c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:15375, start at 2024-05-06 16:58:01.32364594 +0800 CST m=+5.822421958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:01.345 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:01.345 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:01.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347f3d800003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:15459, start at 2024-05-06 16:58:01.441486522 +0800 CST m=+5.883339595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:01.452 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:01.440 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:01.440 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table foreign_key.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later + set +x + tso='449573259253645313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573259253645313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:04 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.29162918.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 16:58:03 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.27022704.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.t55 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2897.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449573259520507905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573259520507905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:05 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.29092911.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table cdc_tiflash_test.multi_data_type not exists for 2-th check, retry later table test.finish_mark not exists for 10-th check, retry later table row_format.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 8-th check, retry later [Mon May 6 16:58:05 CST 2024] <<<<<< START cdc server in kafka_column_selector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.1681016812.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.t55 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449573259882528769 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573259882528769 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:06 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.29322934.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:07 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a42d12b2-1269-4a4a-82bc-aaa7586caf0d {"id":"a42d12b2-1269-4a4a-82bc-aaa7586caf0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f902fd3 a42d12b2-1269-4a4a-82bc-aaa7586caf0d /tidb/cdc/default/default/upstream/7365808219174360230 {"id":7365808219174360230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a42d12b2-1269-4a4a-82bc-aaa7586caf0d {"id":"a42d12b2-1269-4a4a-82bc-aaa7586caf0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f902fd3 a42d12b2-1269-4a4a-82bc-aaa7586caf0d /tidb/cdc/default/default/upstream/7365808219174360230 {"id":7365808219174360230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a42d12b2-1269-4a4a-82bc-aaa7586caf0d {"id":"a42d12b2-1269-4a4a-82bc-aaa7586caf0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f902fd3 a42d12b2-1269-4a4a-82bc-aaa7586caf0d /tidb/cdc/default/default/upstream/7365808219174360230 {"id":7365808219174360230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.finish_mark not exists for 11-th check, retry later [Mon May 6 16:58:07 CST 2024] <<<<<< START kafka consumer in gc_safepoint case >>>>>> 0 table row_format.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:07 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13530314-9014-4f99-83cf-d0be355a17ae {"id":"13530314-9014-4f99-83cf-d0be355a17ae","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f96d9d5 13530314-9014-4f99-83cf-d0be355a17ae /tidb/cdc/default/default/upstream/7365808216303432435 {"id":7365808216303432435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13530314-9014-4f99-83cf-d0be355a17ae {"id":"13530314-9014-4f99-83cf-d0be355a17ae","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f96d9d5 13530314-9014-4f99-83cf-d0be355a17ae /tidb/cdc/default/default/upstream/7365808216303432435 {"id":7365808216303432435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13530314-9014-4f99-83cf-d0be355a17ae {"id":"13530314-9014-4f99-83cf-d0be355a17ae","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f96d9d5 13530314-9014-4f99-83cf-d0be355a17ae /tidb/cdc/default/default/upstream/7365808216303432435 {"id":7365808216303432435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2969.out cli changefeed create --start-ts=449573259253645313 '--sink-uri=kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-2366?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table foreign_key.finish_mark not exists for 3-th check, retry later Create changefeed successfully! ID: e6707721-9a2f-434e-b615-a8f35ea4e59c Info: {"upstream_id":7365808216303432435,"namespace":"default","id":"e6707721-9a2f-434e-b615-a8f35ea4e59c","sink_uri":"kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-2366?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:07.854622154+08:00","start_ts":449573259253645313,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573259253645313,"checkpoint_ts":449573259253645313,"checkpoint_time":"2024-05-06 16:58:02.773"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table cdc_tiflash_test.multi_data_type exists check diff successfully check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... table multi_tables_ddl_test.t55 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:08 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92d446bf-c091-4f49-b054-b6f4eef52c0a {"id":"92d446bf-c091-4f49-b054-b6f4eef52c0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f98dbd1 92d446bf-c091-4f49-b054-b6f4eef52c0a /tidb/cdc/default/default/upstream/7365808220801092564 {"id":7365808220801092564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92d446bf-c091-4f49-b054-b6f4eef52c0a {"id":"92d446bf-c091-4f49-b054-b6f4eef52c0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f98dbd1 92d446bf-c091-4f49-b054-b6f4eef52c0a /tidb/cdc/default/default/upstream/7365808220801092564 {"id":7365808220801092564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92d446bf-c091-4f49-b054-b6f4eef52c0a {"id":"92d446bf-c091-4f49-b054-b6f4eef52c0a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1f98dbd1 92d446bf-c091-4f49-b054-b6f4eef52c0a /tidb/cdc/default/default/upstream/7365808220801092564 {"id":7365808220801092564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2963.out cli changefeed create --start-ts=449573259520507905 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resolve-lock-test-2592?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table ddl_manager.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: 1d15e40c-c87d-4001-93c3-3a46d58a9051 Info: {"upstream_id":7365808220801092564,"namespace":"default","id":"1d15e40c-c87d-4001-93c3-3a46d58a9051","sink_uri":"kafka://127.0.0.1:9092/ticdc-resolve-lock-test-2592?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:08.907041612+08:00","start_ts":449573259520507905,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573259520507905,"checkpoint_ts":449573259520507905,"checkpoint_time":"2024-05-06 16:58:03.791"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Mon May 6 16:58:09 CST 2024] <<<<<< START kafka consumer in many_pk_or_uk case >>>>>> go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 table test.finish_mark not exists for 12-th check, retry later wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:09 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e97b885e-7422-493a-ada3-f4f2c0ccd39f {"id":"e97b885e-7422-493a-ada3-f4f2c0ccd39f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985886} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1faa12cd e97b885e-7422-493a-ada3-f4f2c0ccd39f /tidb/cdc/default/default/upstream/7365808236525044226 {"id":7365808236525044226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e97b885e-7422-493a-ada3-f4f2c0ccd39f {"id":"e97b885e-7422-493a-ada3-f4f2c0ccd39f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985886} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1faa12cd e97b885e-7422-493a-ada3-f4f2c0ccd39f /tidb/cdc/default/default/upstream/7365808236525044226 {"id":7365808236525044226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e97b885e-7422-493a-ada3-f4f2c0ccd39f {"id":"e97b885e-7422-493a-ada3-f4f2c0ccd39f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985886} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1faa12cd e97b885e-7422-493a-ada3-f4f2c0ccd39f /tidb/cdc/default/default/upstream/7365808236525044226 {"id":7365808236525044226,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.cli.16872.out cli changefeed create --start-ts=449573260072058882 '--sink-uri=kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json&partition-num=1&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/conf/changefeed.toml Create changefeed successfully! ID: test Info: {"upstream_id":7365808236525044226,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json\u0026partition-num=1\u0026enable-tidb-extension=true","create_time":"2024-05-06T16:58:09.496182941+08:00","start_ts":449573260072058882,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.t1"],"columns":["a","b"]},{"matcher":["test.*"],"columns":["*","!b"]},{"matcher":["test1.t1"],"columns":["column*","!column1"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573260072058882,"checkpoint_ts":449573260072058882,"checkpoint_time":"2024-05-06 16:58:05.895"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 table foreign_key.finish_mark not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:09 CST 2024] <<<<<< run test case tiflash success! >>>>>> go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table row_format.finish_mark not exists for 3-th check, retry later go: downloading golang.org/x/text v0.14.0 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:10 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a965ef61-d066-4d21-af82-12d6e22b78d6 {"id":"a965ef61-d066-4d21-af82-12d6e22b78d6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985888} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fa035d9 a965ef61-d066-4d21-af82-12d6e22b78d6 /tidb/cdc/default/default/upstream/7365808232528657395 {"id":7365808232528657395,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a965ef61-d066-4d21-af82-12d6e22b78d6 {"id":"a965ef61-d066-4d21-af82-12d6e22b78d6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985888} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fa035d9 a965ef61-d066-4d21-af82-12d6e22b78d6 /tidb/cdc/default/default/upstream/7365808232528657395 {"id":7365808232528657395,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a965ef61-d066-4d21-af82-12d6e22b78d6 {"id":"a965ef61-d066-4d21-af82-12d6e22b78d6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985888} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fa035d9 a965ef61-d066-4d21-af82-12d6e22b78d6 /tidb/cdc/default/default/upstream/7365808232528657395 {"id":7365808232528657395,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2973.out cli changefeed create --start-ts=449573259882528769 '--sink-uri=kafka://127.0.0.1:9092/ticdc-default-value-test-21312?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + set +x [Mon May 6 16:58:10 CST 2024] <<<<<< START kafka consumer in resolve_lock case >>>>>> go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 table ddl_manager.finish_mark not exists for 10-th check, retry later check diff failed 2-th time, retry later go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading golang.org/x/sys v0.19.0 Create changefeed successfully! ID: 22c7c0c3-cf9f-4a3b-9d66-d9e4e87a50f9 Info: {"upstream_id":7365808232528657395,"namespace":"default","id":"22c7c0c3-cf9f-4a3b-9d66-d9e4e87a50f9","sink_uri":"kafka://127.0.0.1:9092/ticdc-default-value-test-21312?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:10.62668419+08:00","start_ts":449573259882528769,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573259882528769,"checkpoint_ts":449573259882528769,"checkpoint_time":"2024-05-06 16:58:05.172"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda table multi_tables_ddl_test.t55 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/text v0.14.0 table test.finish_mark exists check diff successfully table foreign_key.finish_mark not exists for 5-th check, retry later Starting build checksum checker... go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading google.golang.org/grpc v1.62.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/sync v0.7.0 go: downloading golang.org/x/time v0.5.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 table row_format.finish_mark not exists for 4-th check, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading go.uber.org/mock v0.4.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading golang.org/x/time v0.5.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading go.uber.org/multierr v1.11.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 + set +x [Mon May 6 16:58:12 CST 2024] <<<<<< START kafka consumer in default_value case >>>>>> go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 table ddl_manager.finish_mark not exists for 11-th check, retry later go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365808219174360230 449573261560250369 449573260407078918 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading go.uber.org/mock v0.4.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/apache/thrift v0.16.0 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da run task successfully go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading k8s.io/apimachinery v0.28.6 check_changefeed_state http://127.0.0.1:2379 2a0430b7-75bd-44c2-846a-93866c191c17 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=2a0430b7-75bd-44c2-846a-93866c191c17 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2a0430b7-75bd-44c2-846a-93866c191c17 -s table row_format.finish_mark not exists for 5-th check, retry later go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b table foreign_key.finish_mark not exists for 6-th check, retry later + info='{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "stopped", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null }' + echo '{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "stopped", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null }' { "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "stopped", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null } ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"2a0430b7-75bd-44c2-846a-93866c191c17",' '"state":' '"stopped",' '"checkpoint_tso":' 449573261822656514, '"checkpoint_time":' '"2024-05-06' '16:58:12.573",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"2a0430b7-75bd-44c2-846a-93866c191c17",' '"state":' '"stopped",' '"checkpoint_tso":' 449573261822656514, '"checkpoint_time":' '"2024-05-06' '16:58:12.573",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365808219174360230 go: downloading github.com/ardielle/ardielle-go v1.5.2 table ddl_manager.finish_mark not exists for 12-th check, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading golang.org/x/tools v0.20.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/kr/pretty v0.3.1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347fe0e4001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:4298, start at 2024-05-06 16:58:11.946032872 +0800 CST m=+5.254397171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:11.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:11.947 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:11.947 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347fe0e4001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:4298, start at 2024-05-06 16:58:11.946032872 +0800 CST m=+5.254397171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:11.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:11.947 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:11.947 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d347fe6500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:4377, start at 2024-05-06 16:58:12.274617842 +0800 CST m=+5.528430961 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:12.281 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:12.244 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:12.244 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 table row_format.finish_mark not exists for 6-th check, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table foreign_key.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 table test.finish_mark not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449573259674386468, "checkpoint_time": "2024-05-06 16:58:04.378", "error": null }' + echo '{ "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449573259674386468, "checkpoint_time": "2024-05-06 16:58:04.378", "error": null }' { "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449573259674386468, "checkpoint_time": "2024-05-06 16:58:04.378", "error": null } ++ echo '{' '"upstream_id":' 7365808193414000926, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449573259674386468, '"checkpoint_time":' '"2024-05-06' '16:58:04.378",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808193414000926, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449573259674386468, '"checkpoint_time":' '"2024-05-06' '16:58:04.378",' '"error":' null '}' ++ jq -r .error.message + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5721.out cli tso query --pd=http://127.0.0.1:2379 + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449573262741471234, "checkpoint_time": "2024-05-06 16:58:16.078", "error": null }' + echo '{ "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449573262741471234, "checkpoint_time": "2024-05-06 16:58:16.078", "error": null }' { "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449573262741471234, "checkpoint_time": "2024-05-06 16:58:16.078", "error": null } ++ echo '{' '"upstream_id":' 7365808193414000926, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449573262741471234, '"checkpoint_time":' '"2024-05-06' '16:58:16.078",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808193414000926, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449573262741471234, '"checkpoint_time":' '"2024-05-06' '16:58:16.078",' '"error":' null '}' ++ jq -r .error.message go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449573259202527237, "checkpoint_time": "2024-05-06 16:58:02.578", "error": { "time": "2024-05-06T16:58:05.115339895+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449573259202527237, "checkpoint_time": "2024-05-06 16:58:02.578", "error": { "time": "2024-05-06T16:58:05.115339895+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7365808193414000926, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449573259202527237, "checkpoint_time": "2024-05-06 16:58:02.578", "error": { "time": "2024-05-06T16:58:05.115339895+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365808193414000926, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449573259202527237, '"checkpoint_time":' '"2024-05-06' '16:58:02.578",' '"error":' '{' '"time":' '"2024-05-06T16:58:05.115339895+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365808193414000926, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449573259202527237, '"checkpoint_time":' '"2024-05-06' '16:58:02.578",' '"error":' '{' '"time":' '"2024-05-06T16:58:05.115339895+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] run task successfully check diff successfully table row_format.finish_mark not exists for 7-th check, retry later check_changefeed_state http://127.0.0.1:2379 2a0430b7-75bd-44c2-846a-93866c191c17 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=2a0430b7-75bd-44c2-846a-93866c191c17 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2a0430b7-75bd-44c2-846a-93866c191c17 -s table foreign_key.finish_mark not exists for 8-th check, retry later + info='{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "normal", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null }' + echo '{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "normal", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null }' { "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "normal", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null } ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"2a0430b7-75bd-44c2-846a-93866c191c17",' '"state":' '"normal",' '"checkpoint_tso":' 449573261822656514, '"checkpoint_time":' '"2024-05-06' '16:58:12.573",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"2a0430b7-75bd-44c2-846a-93866c191c17",' '"state":' '"normal",' '"checkpoint_tso":' 449573261822656514, '"checkpoint_time":' '"2024-05-06' '16:58:12.573",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365808219174360230 449573261822656513 449573261822656514 wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 14-th check, retry later wait process cdc.test exit for 2-th time... go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 + set +x + tso='449573263074131969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573263074131969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x run task successfully wait process cdc.test exit for 3-th time... check_changefeed_state http://127.0.0.1:2379 2a0430b7-75bd-44c2-846a-93866c191c17 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=2a0430b7-75bd-44c2-846a-93866c191c17 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2a0430b7-75bd-44c2-846a-93866c191c17 -s + info='{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "stopped", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null }' + echo '{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "stopped", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null }' { "upstream_id": 7365808219174360230, "namespace": "default", "id": "2a0430b7-75bd-44c2-846a-93866c191c17", "state": "stopped", "checkpoint_tso": 449573261822656514, "checkpoint_time": "2024-05-06 16:58:12.573", "error": null } ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"2a0430b7-75bd-44c2-846a-93866c191c17",' '"state":' '"stopped",' '"checkpoint_tso":' 449573261822656514, '"checkpoint_time":' '"2024-05-06' '16:58:12.573",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"2a0430b7-75bd-44c2-846a-93866c191c17",' '"state":' '"stopped",' '"checkpoint_tso":' 449573261822656514, '"checkpoint_time":' '"2024-05-06' '16:58:12.573",' '"error":' null '}' ++ jq -r .error.message cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:19 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> table row_format.finish_mark not exists for 8-th check, retry later + message=null + [[ ! null =~ null ]] run task successfully go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd ***************** properties ***************** "readallfields"="true" "operationcount"="0" "recordcount"="20" "scanproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "readproportion"="0" "dotransactions"="false" "mysql.db"="changefeed_error" "requestdistribution"="uniform" "workload"="core" "mysql.port"="4000" "updateproportion"="0" "threadcount"="4" "mysql.user"="root" ********************************************** check_changefeed_state http://127.0.0.1:2379 ae383573-c3b4-4449-940e-4831839b768a normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=ae383573-c3b4-4449-940e-4831839b768a + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c ae383573-c3b4-4449-940e-4831839b768a -s + info='{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "ae383573-c3b4-4449-940e-4831839b768a", "state": "normal", "checkpoint_tso": 449573263696723969, "checkpoint_time": "2024-05-06 16:58:19.722", "error": null }' + echo '{ "upstream_id": 7365808219174360230, "namespace": "default", "id": "ae383573-c3b4-4449-940e-4831839b768a", "state": "normal", "checkpoint_tso": 449573263696723969, "checkpoint_time": "2024-05-06 16:58:19.722", "error": null }' { "upstream_id": 7365808219174360230, "namespace": "default", "id": "ae383573-c3b4-4449-940e-4831839b768a", "state": "normal", "checkpoint_tso": 449573263696723969, "checkpoint_time": "2024-05-06 16:58:19.722", "error": null } ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"ae383573-c3b4-4449-940e-4831839b768a",' '"state":' '"normal",' '"checkpoint_tso":' 449573263696723969, '"checkpoint_time":' '"2024-05-06' '16:58:19.722",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808219174360230, '"namespace":' '"default",' '"id":' '"ae383573-c3b4-4449-940e-4831839b768a",' '"state":' '"normal",' '"checkpoint_tso":' 449573263696723969, '"checkpoint_time":' '"2024-05-06' '16:58:19.722",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365808219174360230 table test.finish_mark not exists for 3-th check, retry later Run finished, takes 302.273063ms INSERT - Takes(s): 0.0, Count: 20, OPS: 424.1, Avg(us): 56300, Min(us): 1208, Max(us): 255077, 95th(us): 256000, 99th(us): 256000 [Mon May 6 16:58:20 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57905792.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 9-th check, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 4-th check, retry later table row_format.finish_mark exists check diff successfully table foreign_key.finish_mark not exists for 10-th check, retry later wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 16-th check, retry later wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:58:23 CST 2024] <<<<<< run test case row_format success! >>>>>> run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:23 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c {"id":"862aed11-7e58-415b-bfa8-8a70302a856c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985900} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc12f4 862aed11-7e58-415b-bfa8-8a70302a856c /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c {"id":"862aed11-7e58-415b-bfa8-8a70302a856c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985900} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc12f4 862aed11-7e58-415b-bfa8-8a70302a856c /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c {"id":"862aed11-7e58-415b-bfa8-8a70302a856c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985900} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc12f4 862aed11-7e58-415b-bfa8-8a70302a856c /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5850.out cli changefeed create --start-ts=449573263074131969 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error Changefeed remove successfully. ID: 2a0430b7-75bd-44c2-846a-93866c191c17 CheckpointTs: 449573261822656514 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-9120?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_forward http://127.0.0.1:2379 7365808219174360230 449573261822656513 449573261822656514 449573263696723969 Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365808283075265434,"namespace":"default","id":"changefeed-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:23.735507872+08:00","start_ts":449573263074131969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573263074131969,"checkpoint_ts":449573263074131969,"checkpoint_time":"2024-05-06 16:58:17.347"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 17-th check, retry later wait process cdc.test exit for 1-th time... run task successfully start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x [Mon May 6 16:58:25 CST 2024] <<<<<< START kafka consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... Changefeed remove successfully. ID: ae383573-c3b4-4449-940e-4831839b768a CheckpointTs: 449573264968384515 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-9120?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_cleared http://127.0.0.1:2379 7365808219174360230 + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449573263074131969, "checkpoint_time": "2024-05-06 16:58:17.347", "error": { "time": "2024-05-06T16:58:23.818284185+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573263074131669 is earlier than or equal to GC safepoint at 449573263074131969" } }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449573263074131969, "checkpoint_time": "2024-05-06 16:58:17.347", "error": { "time": "2024-05-06T16:58:23.818284185+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573263074131669 is earlier than or equal to GC safepoint at 449573263074131969" } }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449573263074131969, "checkpoint_time": "2024-05-06 16:58:17.347", "error": { "time": "2024-05-06T16:58:23.818284185+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573263074131669 is earlier than or equal to GC safepoint at 449573263074131969" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449573263074131969, '"checkpoint_time":' '"2024-05-06' '16:58:17.347",' '"error":' '{' '"time":' '"2024-05-06T16:58:23.818284185+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449573263074131669 is earlier than or equal to GC safepoint at '449573263074131969"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449573263074131969, '"checkpoint_time":' '"2024-05-06' '16:58:17.347",' '"error":' '{' '"time":' '"2024-05-06T16:58:23.818284185+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449573263074131669 is earlier than or equal to GC safepoint at '449573263074131969"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573263074131669 is earlier than or equal to GC safepoint at 449573263074131969' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449573263074131669 is earlier than or equal to GC safepoint at 449573263074131969 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5972.out cli changefeed resume -c changefeed-error run task successfully wait process cdc.test exit for 3-th time... PASS cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:26 CST 2024] <<<<<< run test case foreign_key success! >>>>>> wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 18-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 6-th check, retry later wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:27 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> + set +x table changefeed_error.usertable not exists for 1-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:24 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> table ddl_manager.finish_mark not exists for 19-th check, retry later table test.finish_mark not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table changefeed_error.usertable exists VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3480e8140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:4392, start at 2024-05-06 16:58:28.762781579 +0800 CST m=+5.572614096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:28.774 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:28.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:28.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later table test.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3480e8140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:4392, start at 2024-05-06 16:58:28.762781579 +0800 CST m=+5.572614096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:28.774 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:28.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:28.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3480e99c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:4479, start at 2024-05-06 16:58:28.846374699 +0800 CST m=+5.598832920 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:28.857 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:28.839 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:28.839 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully wait process cdc.test exit for 1-th time... ***************** properties ***************** "updateproportion"="0" "operationcount"="0" "mysql.db"="changefeed_error" "threadcount"="4" "insertproportion"="0" "scanproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "dotransactions"="false" "requestdistribution"="uniform" "recordcount"="20" "mysql.port"="4000" "mysql.user"="root" "workload"="core" ********************************************** Run finished, takes 5.392336ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5005.6, Avg(us): 902, Min(us): 478, Max(us): 1895, 95th(us): 2000, 99th(us): 2000 check diff successfully {"id":"862aed11-7e58-415b-bfa8-8a70302a856c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985900} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c + [[ /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 21-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:33 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:58:33 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.58565858.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 22-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c + [[ /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 23-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: kafka... <<================= + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:36 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a6d0d56-d2c6-4136-b5a0-4d026610d52a {"id":"2a6d0d56-d2c6-4136-b5a0-4d026610d52a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d201065cd 2a6d0d56-d2c6-4136-b5a0-4d026610d52a /tidb/cdc/default/default/upstream/7365808355794922320 {"id":7365808355794922320,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a6d0d56-d2c6-4136-b5a0-4d026610d52a {"id":"2a6d0d56-d2c6-4136-b5a0-4d026610d52a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d201065cd 2a6d0d56-d2c6-4136-b5a0-4d026610d52a /tidb/cdc/default/default/upstream/7365808355794922320 {"id":7365808355794922320,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2a6d0d56-d2c6-4136-b5a0-4d026610d52a {"id":"2a6d0d56-d2c6-4136-b5a0-4d026610d52a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d201065cd 2a6d0d56-d2c6-4136-b5a0-4d026610d52a /tidb/cdc/default/default/upstream/7365808355794922320 {"id":7365808355794922320,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 16:58:36 CST 2024] <<<<<< START kafka consumer in changefeed_reconstruct case >>>>>> ***************** properties ***************** "scanproportion"="0" "workload"="core" "recordcount"="50" "insertproportion"="0" "readproportion"="0" "dotransactions"="false" "mysql.port"="4000" "mysql.db"="changefeed_reconstruct" "readallfields"="true" "mysql.host"="127.0.0.1" "operationcount"="0" "requestdistribution"="uniform" "updateproportion"="0" "threadcount"="4" "mysql.user"="root" ********************************************** Run finished, takes 20.480161ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3052.6, Avg(us): 1627, Min(us): 1064, Max(us): 5012, 95th(us): 5000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/multi_topics_v2 Starting Upstream PD... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c + [[ /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/862aed11-7e58-415b-bfa8-8a70302a856c + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 24-th check, retry later table changefeed_reconstruct.usertable not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348164780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw, pid:4289, start at 2024-05-06 16:58:36.740542948 +0800 CST m=+5.662961955 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:36.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:36.752 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:36.752 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348164780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw, pid:4289, start at 2024-05-06 16:58:36.740542948 +0800 CST m=+5.662961955 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:36.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:36.752 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:36.752 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348165dc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4tqk4-s4vvw, pid:4365, start at 2024-05-06 16:58:36.812239551 +0800 CST m=+5.680901561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:36.819 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:36.791 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:36.791 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 25-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table changefeed_reconstruct.usertable exists [Mon May 6 16:58:41 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.56895691.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process 5861 exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process 5861 exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 26-th check, retry later wait process 5861 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5861) - No such process wait process 5861 exit for 4-th time... process 5861 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:58:43 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.61386140.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 27-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:44 GMT < Content-Length: 859 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85999670-4d2e-4339-8109-407184bb4c52 {"id":"85999670-4d2e-4339-8109-407184bb4c52","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985921} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d202f1233 85999670-4d2e-4339-8109-407184bb4c52 /tidb/cdc/default/default/upstream/7365808384524906393 {"id":7365808384524906393,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85999670-4d2e-4339-8109-407184bb4c52 {"id":"85999670-4d2e-4339-8109-407184bb4c52","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985921} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d202f1233 85999670-4d2e-4339-8109-407184bb4c52 /tidb/cdc/default/default/upstream/7365808384524906393 {"id":7365808384524906393,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85999670-4d2e-4339-8109-407184bb4c52 {"id":"85999670-4d2e-4339-8109-407184bb4c52","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985921} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d202f1233 85999670-4d2e-4339-8109-407184bb4c52 /tidb/cdc/default/default/upstream/7365808384524906393 {"id":7365808384524906393,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Create changefeed successfully! ID: a259c6ae-33a2-4b76-a1fe-ebb04ca5e5a2 Info: {"upstream_id":7365808384524906393,"namespace":"default","id":"a259c6ae-33a2-4b76-a1fe-ebb04ca5e5a2","sink_uri":"kafka://127.0.0.1:9092/ticdc-region-merge-test-23695?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:44.765698181+08:00","start_ts":449573270218604546,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573270218604546,"checkpoint_ts":449573270218604546,"checkpoint_time":"2024-05-06 16:58:44.601"} [Mon May 6 16:58:44 CST 2024] <<<<<< START kafka consumer in region_merge case >>>>>> Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 16:58:43 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62386240.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... split_and_random_merge scale: 20 table ddl_manager.finish_mark not exists for 28-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/276d2592-cacd-488b-b098-41a2b95afdb7 {UpstreamID:7365808355794922320 Namespace:default ID:276d2592-cacd-488b-b098-41a2b95afdb7 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-16391?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:58:36.901833493 +0800 CST StartTs:449573268158152708 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003190750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573268197474307} {CheckpointTs:449573270386376710 MinTableBarrierTs:449573270648520709 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573270648520709, checkpointTs: 449573270648520709, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fe4330a-8ac5-49b6-a446-ea314d751ae0 {"id":"9fe4330a-8ac5-49b6-a446-ea314d751ae0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20106668 9fe4330a-8ac5-49b6-a446-ea314d751ae0 /tidb/cdc/default/default/changefeed/info/276d2592-cacd-488b-b098-41a2b95afdb7 {"upstream-id":7365808355794922320,"namespace":"default","changefeed-id":"276d2592-cacd-488b-b098-41a2b95afdb7","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-16391?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:58:36.901833493+08:00","start-ts":449573268158152708,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573268197474307} /tidb/cdc/default/default/changefeed/status/276d2592-cacd-488b-b098-41a2b95afdb7 {"checkpoint-ts":449573270648520709,"min-table-barrier-ts":449573270648520709,"admin-job-type":0} /tidb/cdc/default/default/task/position/9fe4330a-8ac5-49b6-a446-ea314d751ae0/276d2592-cacd-488b-b098-41a2b95afdb7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808355794922320 {"id":7365808355794922320,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/276d2592-cacd-488b-b098-41a2b95afdb7 {UpstreamID:7365808355794922320 Namespace:default ID:276d2592-cacd-488b-b098-41a2b95afdb7 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-16391?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:58:36.901833493 +0800 CST StartTs:449573268158152708 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003190750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573268197474307} {CheckpointTs:449573270386376710 MinTableBarrierTs:449573270648520709 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573270648520709, checkpointTs: 449573270648520709, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fe4330a-8ac5-49b6-a446-ea314d751ae0 {"id":"9fe4330a-8ac5-49b6-a446-ea314d751ae0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20106668 9fe4330a-8ac5-49b6-a446-ea314d751ae0 /tidb/cdc/default/default/changefeed/info/276d2592-cacd-488b-b098-41a2b95afdb7 {"upstream-id":7365808355794922320,"namespace":"default","changefeed-id":"276d2592-cacd-488b-b098-41a2b95afdb7","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-16391?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:58:36.901833493+08:00","start-ts":449573268158152708,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573268197474307} /tidb/cdc/default/default/changefeed/status/276d2592-cacd-488b-b098-41a2b95afdb7 {"checkpoint-ts":449573270648520709,"min-table-barrier-ts":449573270648520709,"admin-job-type":0} /tidb/cdc/default/default/task/position/9fe4330a-8ac5-49b6-a446-ea314d751ae0/276d2592-cacd-488b-b098-41a2b95afdb7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808355794922320 + grep -q 'failed to get info:' {"id":7365808355794922320,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/276d2592-cacd-488b-b098-41a2b95afdb7 {UpstreamID:7365808355794922320 Namespace:default ID:276d2592-cacd-488b-b098-41a2b95afdb7 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-16391?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 16:58:36.901833493 +0800 CST StartTs:449573268158152708 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003190750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-22-ge9f061d7a Epoch:449573268197474307} {CheckpointTs:449573270386376710 MinTableBarrierTs:449573270648520709 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449573270648520709, checkpointTs: 449573270648520709, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fe4330a-8ac5-49b6-a446-ea314d751ae0 {"id":"9fe4330a-8ac5-49b6-a446-ea314d751ae0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20106668 9fe4330a-8ac5-49b6-a446-ea314d751ae0 /tidb/cdc/default/default/changefeed/info/276d2592-cacd-488b-b098-41a2b95afdb7 {"upstream-id":7365808355794922320,"namespace":"default","changefeed-id":"276d2592-cacd-488b-b098-41a2b95afdb7","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-16391?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:58:36.901833493+08:00","start-ts":449573268158152708,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573268197474307} /tidb/cdc/default/default/changefeed/status/276d2592-cacd-488b-b098-41a2b95afdb7 {"checkpoint-ts":449573270648520709,"min-table-barrier-ts":449573270648520709,"admin-job-type":0} /tidb/cdc/default/default/task/position/9fe4330a-8ac5-49b6-a446-ea314d751ae0/276d2592-cacd-488b-b098-41a2b95afdb7 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365808355794922320 {"id":7365808355794922320,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31bcf998-7728-4626-80a2-ff7a13088797 {"id":"31bcf998-7728-4626-80a2-ff7a13088797","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc13e8 31bcf998-7728-4626-80a2-ff7a13088797 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365808283075265434,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:58:23.735507872+08:00","start-ts":449573263074131969,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T16:58:44.564778097+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573270204448771} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449573266586861573,"min-table-barrier-ts":449573266849005578,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31bcf998-7728-4626-80a2-ff7a13088797 {"id":"31bcf998-7728-4626-80a2-ff7a13088797","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc13e8 31bcf998-7728-4626-80a2-ff7a13088797 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365808283075265434,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:58:23.735507872+08:00","start-ts":449573263074131969,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T16:58:44.564778097+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573270204448771} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449573266586861573,"min-table-barrier-ts":449573266849005578,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365808283075265434 + grep -q 'failed to get info:' {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/31bcf998-7728-4626-80a2-ff7a13088797 {"id":"31bcf998-7728-4626-80a2-ff7a13088797","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc13e8 31bcf998-7728-4626-80a2-ff7a13088797 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365808283075265434,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T16:58:23.735507872+08:00","start-ts":449573263074131969,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T16:58:44.564778097+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-22-ge9f061d7a","epoch":449573270204448771} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449573266586861573,"min-table-barrier-ts":449573266849005578,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449573266586861573, "checkpoint_time": "2024-05-06 16:58:30.747", "error": { "time": "2024-05-06T16:58:44.564778097+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449573266586861573, "checkpoint_time": "2024-05-06 16:58:30.747", "error": { "time": "2024-05-06T16:58:44.564778097+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449573266586861573, "checkpoint_time": "2024-05-06 16:58:30.747", "error": { "time": "2024-05-06T16:58:44.564778097+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573266586861573, '"checkpoint_time":' '"2024-05-06' '16:58:30.747",' '"error":' '{' '"time":' '"2024-05-06T16:58:44.564778097+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573266586861573, '"checkpoint_time":' '"2024-05-06' '16:58:30.747",' '"error":' '{' '"time":' '"2024-05-06T16:58:44.564778097+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6341.out cli changefeed remove -c changefeed-error start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "id": "9fe4330a-8ac5-49b6-a446-ea314d751ae0", "cluster-id": "default" run task successfully Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449573266586861573 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 29-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release capture_id: 9fe4330a-8ac5-49b6-a446-ea314d751ae0 check_processor_table_count http://127.0.0.1:2379 276d2592-cacd-488b-b098-41a2b95afdb7 9fe4330a-8ac5-49b6-a446-ea314d751ae0 1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34821d440010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:4121, start at 2024-05-06 16:58:48.544876094 +0800 CST m=+5.177294863 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_processor_table_count http://127.0.0.1:2379 276d2592-cacd-488b-b098-41a2b95afdb7 9fe4330a-8ac5-49b6-a446-ea314d751ae0 0 wait process cdc.test exit for 1-th time... table count 1 does equal to expected count 0 run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 30-th check, retry later wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 16:58:51 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e Starting Upstream TiDB... + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.64386440.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34821d440010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:4121, start at 2024-05-06 16:58:48.544876094 +0800 CST m=+5.177294863 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.553 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34821c500018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:4198, start at 2024-05-06 16:58:48.506905615 +0800 CST m=+5.088570107 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.514 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.519 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.519 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348223380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:9057, start at 2024-05-06 16:58:48.931703091 +0800 CST m=+5.177124038 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.910 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.910 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348223380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:9057, start at 2024-05-06 16:58:48.931703091 +0800 CST m=+5.177124038 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:48.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:48.910 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:48.910 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348224ac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-t2mdb-7sn4x, pid:9141, start at 2024-05-06 16:58:49.045252347 +0800 CST m=+5.240055659 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:49.053 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:49.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:49.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_processor_table_count http://127.0.0.1:2379 276d2592-cacd-488b-b098-41a2b95afdb7 9fe4330a-8ac5-49b6-a446-ea314d751ae0 0 *************************** 1. row *************************** count(distinct region_id): 1 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-vgp9b-w63gj Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-676wd-f31zb Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-n41ng-sn8j4 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1870-chmlt-fchdr table ddl_manager.finish_mark not exists for 31-th check, retry later run task successfully ***************** properties ***************** "recordcount"="50" "insertproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "mysql.user"="root" "updateproportion"="0" "readproportion"="0" "mysql.port"="4000" "operationcount"="0" "readallfields"="true" "threadcount"="4" "workload"="core" "mysql.db"="changefeed_reconstruct" "scanproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 20.274826ms INSERT - Takes(s): 0.0, Count: 45, OPS: 2909.5, Avg(us): 1362, Min(us): 1066, Max(us): 4666, 95th(us): 2000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic_avro Starting Upstream PD... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5512.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10539.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:54 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd796cd1-31c4-4fef-99a8-b4a11d1fedd2 {"id":"dd796cd1-31c4-4fef-99a8-b4a11d1fedd2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985931} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc1437 dd796cd1-31c4-4fef-99a8-b4a11d1fedd2 /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd796cd1-31c4-4fef-99a8-b4a11d1fedd2 {"id":"dd796cd1-31c4-4fef-99a8-b4a11d1fedd2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985931} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc1437 dd796cd1-31c4-4fef-99a8-b4a11d1fedd2 /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd796cd1-31c4-4fef-99a8-b4a11d1fedd2 {"id":"dd796cd1-31c4-4fef-99a8-b4a11d1fedd2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985931} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc1437 dd796cd1-31c4-4fef-99a8-b4a11d1fedd2 /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6494.out cli changefeed create --start-ts=449573263074131969 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365808283075265434,"namespace":"default","id":"changefeed-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:54.766270178+08:00","start_ts":449573263074131969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573263074131969,"checkpoint_ts":449573263074131969,"checkpoint_time":"2024-05-06 16:58:17.347"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later + set +x + tso='449573272677253121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573272677253121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:55 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.55595561.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449573272790237185 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573272790237185 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:58:55 CST 2024] <<<<<< START cdc server in multi_topics_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.1057410576.out server --log-file /tmp/tidb_cdc_test/multi_topics_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x split_and_random_merge scale: 40 table ddl_manager.finish_mark not exists for 32-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348291800004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g, pid:4413, start at 2024-05-06 16:58:55.971828367 +0800 CST m=+5.408442327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:55.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:55.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:55.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 579 0 --:--:-- --:--:-- --:--:-- 581 + info='{"state":"normal","resolved_ts":449573263074131969,"checkpoint_ts":449573263074131969}' + echo '{"state":"normal","resolved_ts":449573263074131969,"checkpoint_ts":449573263074131969}' {"state":"normal","resolved_ts":449573263074131969,"checkpoint_ts":449573263074131969} ++ echo '{"state":"normal","resolved_ts":449573263074131969,"checkpoint_ts":449573263074131969}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 33-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348291800004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g, pid:4413, start at 2024-05-06 16:58:55.971828367 +0800 CST m=+5.408442327 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:55.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:55.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:55.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348291840014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-x9nnr-9cq2g, pid:4495, start at 2024-05-06 16:58:56.000557026 +0800 CST m=+5.382166639 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:56.007 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:55.969 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:55.969 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:58 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53e1d1da-ca1f-4c07-81d5-1a16d14f0955 {"id":"53e1d1da-ca1f-4c07-81d5-1a16d14f0955","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985935} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205f3ed4 53e1d1da-ca1f-4c07-81d5-1a16d14f0955 /tidb/cdc/default/default/upstream/7365808439779387792 {"id":7365808439779387792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53e1d1da-ca1f-4c07-81d5-1a16d14f0955 {"id":"53e1d1da-ca1f-4c07-81d5-1a16d14f0955","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985935} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205f3ed4 53e1d1da-ca1f-4c07-81d5-1a16d14f0955 /tidb/cdc/default/default/upstream/7365808439779387792 {"id":7365808439779387792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53e1d1da-ca1f-4c07-81d5-1a16d14f0955 {"id":"53e1d1da-ca1f-4c07-81d5-1a16d14f0955","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985935} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d205f3ed4 53e1d1da-ca1f-4c07-81d5-1a16d14f0955 /tidb/cdc/default/default/upstream/7365808439779387792 {"id":7365808439779387792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5610.out cli changefeed create --start-ts=449573272677253121 '--sink-uri=kafka://127.0.0.1:9092/ticdc-drop-tables-test-31908?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 125b8515-24fd-443e-af0f-c030082fca7c Info: {"upstream_id":7365808439779387792,"namespace":"default","id":"125b8515-24fd-443e-af0f-c030082fca7c","sink_uri":"kafka://127.0.0.1:9092/ticdc-drop-tables-test-31908?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:58:58.96551795+08:00","start_ts":449573272677253121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573272677253121,"checkpoint_ts":449573272677253121,"checkpoint_time":"2024-05-06 16:58:53.980"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:58:58 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a4ff05-4d72-463f-a698-df52b277b40e {"id":"97a4ff05-4d72-463f-a698-df52b277b40e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2060c1d5 97a4ff05-4d72-463f-a698-df52b277b40e /tidb/cdc/default/default/upstream/7365808442310252812 {"id":7365808442310252812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a4ff05-4d72-463f-a698-df52b277b40e {"id":"97a4ff05-4d72-463f-a698-df52b277b40e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2060c1d5 97a4ff05-4d72-463f-a698-df52b277b40e /tidb/cdc/default/default/upstream/7365808442310252812 {"id":7365808442310252812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a4ff05-4d72-463f-a698-df52b277b40e {"id":"97a4ff05-4d72-463f-a698-df52b277b40e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2060c1d5 97a4ff05-4d72-463f-a698-df52b277b40e /tidb/cdc/default/default/upstream/7365808442310252812 {"id":7365808442310252812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10634.out cli changefeed create --start-ts=449573272790237185 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/conf/changefeed.toml cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 16:58:59 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2306 0 --:--:-- --:--:-- --:--:-- 2321 + info='{"state":"warning","resolved_ts":449573263467610113,"checkpoint_ts":449573263467610113,"last_warning":{"time":"2024-05-06T16:58:56.998415711+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449573263467610113,"checkpoint_ts":449573263467610113,"last_warning":{"time":"2024-05-06T16:58:56.998415711+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449573263467610113,"checkpoint_ts":449573263467610113,"last_warning":{"time":"2024-05-06T16:58:56.998415711+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449573263467610113,"checkpoint_ts":449573263467610113,"last_warning":{"time":"2024-05-06T16:58:56.998415711+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449573263467610113,"checkpoint_ts":449573263467610113,"last_warning":{"time":"2024-05-06T16:58:56.998415711+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6552.out cli changefeed remove -c changefeed-error-1 Create changefeed successfully! ID: a465922d-e491-4e2c-b765-9ba6f2ccf17a Info: {"upstream_id":7365808442310252812,"namespace":"default","id":"a465922d-e491-4e2c-b765-9ba6f2ccf17a","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-06T16:58:59.421050001+08:00","start_ts":449573272790237185,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573272790237185,"checkpoint_ts":449573272790237185,"checkpoint_time":"2024-05-06 16:58:54.411"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449573263467610113 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 16:59:00 CST 2024] <<<<<< START kafka consumer in drop_many_tables case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:59:00 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.59025904.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table ddl_manager.finish_mark not exists for 34-th check, retry later table drop_tables.c not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482c7dc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb, pid:4424, start at 2024-05-06 16:58:59.487224875 +0800 CST m=+7.861546520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:59.508 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:59.496 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:59.496 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482c7dc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb, pid:4424, start at 2024-05-06 16:58:59.487224875 +0800 CST m=+7.861546520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:59.508 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:59.496 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:59.496 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482c8b80011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-2v9k0-l13lb, pid:4453, start at 2024-05-06 16:58:59.542413083 +0800 CST m=+7.806705401 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:59.559 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:59.552 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:59.552 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482c6740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6, pid:3888, start at 2024-05-06 16:58:59.404456247 +0800 CST m=+7.308147951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:59.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:59.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:59.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482c6740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6, pid:3888, start at 2024-05-06 16:58:59.404456247 +0800 CST m=+7.308147951 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:59.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:59.407 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:59.407 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3482c808000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-dg389-x55b6, pid:3938, start at 2024-05-06 16:58:59.479303392 +0800 CST m=+7.276488563 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:00:59.496 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:58:59.509 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:48:59.509 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + set +x TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 35-th check, retry later *************************** 1. row *************************** count(distinct region_id): 12 wait process cdc.test exit for 1-th time... table drop_tables.c not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:03 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f0e8bae-7800-4982-a725-6fefad92ff2d {"id":"3f0e8bae-7800-4982-a725-6fefad92ff2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d207963cd 3f0e8bae-7800-4982-a725-6fefad92ff2d /tidb/cdc/default/default/upstream/7365808470754557658 {"id":7365808470754557658,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f0e8bae-7800-4982-a725-6fefad92ff2d {"id":"3f0e8bae-7800-4982-a725-6fefad92ff2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d207963cd 3f0e8bae-7800-4982-a725-6fefad92ff2d /tidb/cdc/default/default/upstream/7365808470754557658 {"id":7365808470754557658,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f0e8bae-7800-4982-a725-6fefad92ff2d {"id":"3f0e8bae-7800-4982-a725-6fefad92ff2d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985941} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d207963cd 3f0e8bae-7800-4982-a725-6fefad92ff2d /tidb/cdc/default/default/upstream/7365808470754557658 {"id":7365808470754557658,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5958.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cdc-test-28094?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only [Mon May 6 16:59:04 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.57815783.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 36-th check, retry later table drop_tables.c not exists for 3-th check, retry later + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:59:04 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.66206622.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5242.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: 3cad3671-0c26-4218-97e6-b33f1dff7054 Info: {"upstream_id":7365808470754557658,"namespace":"default","id":"3cad3671-0c26-4218-97e6-b33f1dff7054","sink_uri":"kafka://127.0.0.1:9092/ticdc-cdc-test-28094?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:59:04.519289598+08:00","start_ts":449573275387297797,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573275387297797,"checkpoint_ts":449573275387297797,"checkpoint_time":"2024-05-06 16:59:04.318"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 16:59:05 CST 2024] <<<<<< START kafka consumer in cdc case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 table ddl_manager.finish_mark not exists for 37-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. + (( i++ )) + (( i <= 50 )) system_tz Asia/Shanghai TiDB Global System Timezone. ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL new_collation_enabled True If the new collations are enabled. Do not edit it. * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... ddl_table_version 3 DDL Table Version. Do not delete. * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) tikv_gc_leader_uuid 63d348310600014 Current GC worker leader UUID. (DO NOT EDIT) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:8750, start at 2024-05-06 16:59:04.133663194 +0800 CST m=+5.316651923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:04.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:04.138 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:04.138 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348310600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:8750, start at 2024-05-06 16:59:04.133663194 +0800 CST m=+5.316651923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:04.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:04.138 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:04.138 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348311ec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:8821, start at 2024-05-06 16:59:04.208494692 +0800 CST m=+5.342568162 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:04.216 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:04.187 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:04.187 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table drop_tables.c not exists for 4-th check, retry later + set +x + tso='449573275620343809 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573275620343809 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 16:59:07 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.52855287.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:07 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef472ccc-1f5e-4c06-9858-d2be5585fb10 {"id":"ef472ccc-1f5e-4c06-9858-d2be5585fb10","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985944} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc14cc ef472ccc-1f5e-4c06-9858-d2be5585fb10 /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef472ccc-1f5e-4c06-9858-d2be5585fb10 {"id":"ef472ccc-1f5e-4c06-9858-d2be5585fb10","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985944} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc14cc ef472ccc-1f5e-4c06-9858-d2be5585fb10 /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef472ccc-1f5e-4c06-9858-d2be5585fb10 {"id":"ef472ccc-1f5e-4c06-9858-d2be5585fb10","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985944} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc14cc ef472ccc-1f5e-4c06-9858-d2be5585fb10 /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6678.out cli changefeed create --start-ts=449573263074131969 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:07 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/596cd6a5-2fb5-4e2d-a252-54221632a538 {"id":"596cd6a5-2fb5-4e2d-a252-54221632a538","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985944} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d207b3fd3 596cd6a5-2fb5-4e2d-a252-54221632a538 /tidb/cdc/default/default/upstream/7365808474799305607 {"id":7365808474799305607,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/596cd6a5-2fb5-4e2d-a252-54221632a538 {"id":"596cd6a5-2fb5-4e2d-a252-54221632a538","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985944} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d207b3fd3 596cd6a5-2fb5-4e2d-a252-54221632a538 /tidb/cdc/default/default/upstream/7365808474799305607 {"id":7365808474799305607,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/596cd6a5-2fb5-4e2d-a252-54221632a538 {"id":"596cd6a5-2fb5-4e2d-a252-54221632a538","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985944} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d207b3fd3 596cd6a5-2fb5-4e2d-a252-54221632a538 /tidb/cdc/default/default/upstream/7365808474799305607 {"id":7365808474799305607,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 16:59:07 CST 2024] <<<<<< START kafka consumer in changefeed_pause_resume case >>>>>> Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365808283075265434,"namespace":"default","id":"changefeed-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:59:07.801326118+08:00","start_ts":449573263074131969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573263074131969,"checkpoint_ts":449573263074131969,"checkpoint_time":"2024-05-06 16:58:17.347"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... split_and_random_merge scale: 80 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading cloud.google.com/go v0.112.2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 table ddl_manager.finish_mark not exists for 38-th check, retry later go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible [Mon May 6 16:59:09 CST 2024] <<<<<< START cdc server in kafka_simple_basic_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.1017310175.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449573263074131969, "checkpoint_time": "2024-05-06 16:58:17.347", "error": { "time": "2024-05-06T16:59:07.884990363+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573263074131969 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449573263074131969, "checkpoint_time": "2024-05-06 16:58:17.347", "error": { "time": "2024-05-06T16:59:07.884990363+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573263074131969 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449573263074131969, "checkpoint_time": "2024-05-06 16:58:17.347", "error": { "time": "2024-05-06T16:59:07.884990363+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573263074131969 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449573263074131969, '"checkpoint_time":' '"2024-05-06' '16:58:17.347",' '"error":' '{' '"time":' '"2024-05-06T16:59:07.884990363+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573263074131969 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449573263074131969, '"checkpoint_time":' '"2024-05-06' '16:58:17.347",' '"error":' '{' '"time":' '"2024-05-06T16:59:07.884990363+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573263074131969 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573263074131969 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449573263074131969 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6767.out cli changefeed remove -c changefeed-error-2 table drop_tables.c not exists for 5-th check, retry later table changefeed_pause_resume.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:10 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58e8df0b-73df-4bcd-88c7-d6c15683b749 {"id":"58e8df0b-73df-4bcd-88c7-d6c15683b749","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985947} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2080edfb 58e8df0b-73df-4bcd-88c7-d6c15683b749 /tidb/cdc/default/default/upstream/7365808471203382325 {"id":7365808471203382325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58e8df0b-73df-4bcd-88c7-d6c15683b749 {"id":"58e8df0b-73df-4bcd-88c7-d6c15683b749","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985947} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2080edfb 58e8df0b-73df-4bcd-88c7-d6c15683b749 /tidb/cdc/default/default/upstream/7365808471203382325 {"id":7365808471203382325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58e8df0b-73df-4bcd-88c7-d6c15683b749 {"id":"58e8df0b-73df-4bcd-88c7-d6c15683b749","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985947} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2080edfb 58e8df0b-73df-4bcd-88c7-d6c15683b749 /tidb/cdc/default/default/upstream/7365808471203382325 {"id":7365808471203382325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5335.out cli changefeed create --start-ts=449573275620343809 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-20134?protocol=open-protocol&partition-num=4&kafka-client-id=ddl_puller_lag&kafka-version=2.4.1&max-message-bytes=10485760' Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449573263074131969 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: c4ad8945-4634-4222-8d73-5afcdc21610f Info: {"upstream_id":7365808471203382325,"namespace":"default","id":"c4ad8945-4634-4222-8d73-5afcdc21610f","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-20134?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=ddl_puller_lag\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:59:10.904114927+08:00","start_ts":449573275620343809,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573275620343809,"checkpoint_ts":449573275620343809,"checkpoint_time":"2024-05-06 16:59:05.207"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table drop_tables.c exists check diff successfully table ddl_manager.finish_mark not exists for 39-th check, retry later + set +x wait process cdc.test exit for 1-th time... table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 1-th time... + set +x [Mon May 6 16:59:12 CST 2024] <<<<<< START kafka consumer in ddl_puller_lag case >>>>>> wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:12 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10bbdfeb-82b6-4884-baf8-cd807210ff06 {"id":"10bbdfeb-82b6-4884-baf8-cd807210ff06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d209980ce 10bbdfeb-82b6-4884-baf8-cd807210ff06 /tidb/cdc/default/default/upstream/7365808505096722156 {"id":7365808505096722156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10bbdfeb-82b6-4884-baf8-cd807210ff06 {"id":"10bbdfeb-82b6-4884-baf8-cd807210ff06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d209980ce 10bbdfeb-82b6-4884-baf8-cd807210ff06 /tidb/cdc/default/default/upstream/7365808505096722156 {"id":7365808505096722156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10bbdfeb-82b6-4884-baf8-cd807210ff06 {"id":"10bbdfeb-82b6-4884-baf8-cd807210ff06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985949} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d209980ce 10bbdfeb-82b6-4884-baf8-cd807210ff06 /tidb/cdc/default/default/upstream/7365808505096722156 {"id":7365808505096722156,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.cli.10232.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-avro-26856?protocol=simple&encoding-format=avro' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/conf/changefeed.toml wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: c76619d4-c649-4605-9aaf-443a8e1b9edd Info: {"upstream_id":7365808505096722156,"namespace":"default","id":"c76619d4-c649-4605-9aaf-443a8e1b9edd","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-avro-26856?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-06T16:59:12.777406432+08:00","start_ts":449573277568598020,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573277568598020,"checkpoint_ts":449573277568598020,"checkpoint_time":"2024-05-06 16:59:12.639"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:59:12 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> table ddl_manager.finish_mark not exists for 40-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:59:13 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.68176819.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table changefeed_pause_resume.t3 exists + set +x start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 41-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:16 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e91199c-15e6-42d7-9e60-f6ca1747c6bb {"id":"6e91199c-15e6-42d7-9e60-f6ca1747c6bb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985953} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc1545 6e91199c-15e6-42d7-9e60-f6ca1747c6bb /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e91199c-15e6-42d7-9e60-f6ca1747c6bb {"id":"6e91199c-15e6-42d7-9e60-f6ca1747c6bb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985953} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc1545 6e91199c-15e6-42d7-9e60-f6ca1747c6bb /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e91199c-15e6-42d7-9e60-f6ca1747c6bb {"id":"6e91199c-15e6-42d7-9e60-f6ca1747c6bb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985953} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d1fcc1545 6e91199c-15e6-42d7-9e60-f6ca1747c6bb /tidb/cdc/default/default/upstream/7365808283075265434 {"id":7365808283075265434,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6872.out cli changefeed create --start-ts=0 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-initialize-error Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365808283075265434,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:59:16.869878328+08:00","start_ts":449573278645485571,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573278645485571,"checkpoint_ts":449573278645485571,"checkpoint_time":"2024-05-06 16:59:16.747"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 42-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573278645485571, "checkpoint_time": "2024-05-06 16:59:16.747", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573278645485571, "checkpoint_time": "2024-05-06 16:59:16.747", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573278645485571, "checkpoint_time": "2024-05-06 16:59:16.747", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573278645485571, '"checkpoint_time":' '"2024-05-06' '16:59:16.747",' '"error":' '{' '"time":' '"2024-05-06T16:59:17.049734224+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later [Mon May 6 16:59:19 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic_avro case >>>>>> check diff failed 2-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573278645485571, "checkpoint_time": "2024-05-06 16:59:16.747", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573278645485571, "checkpoint_time": "2024-05-06 16:59:16.747", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449573278645485571, "checkpoint_time": "2024-05-06 16:59:16.747", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449573278645485571, '"checkpoint_time":' '"2024-05-06' '16:59:16.747",' '"error":' '{' '"time":' '"2024-05-06T16:59:17.049734224+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 44-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table test1.finishmark exists [2024/05/06 16:59:15.254 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/06 16:59:15.257 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/06 16:59:15.262 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/06 16:59:15.268 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/06 16:59:15.271 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/06 16:59:15.271 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=23.947289ms] [2024/05/06 16:59:15.274 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/06 16:59:15.276 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/06 16:59:15.278 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/06 16:59:15.284 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/06 16:59:15.286 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/06 16:59:15.286 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=14.788929ms] [2024/05/06 16:59:15.286 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:59:16 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> table ddl_manager.finish_mark not exists for 45-th check, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573280493600773, "checkpoint_time": "2024-05-06 16:59:23.797", "error": null }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573280493600773, "checkpoint_time": "2024-05-06 16:59:23.797", "error": null }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573280493600773, "checkpoint_time": "2024-05-06 16:59:23.797", "error": null } ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573280493600773, '"checkpoint_time":' '"2024-05-06' '16:59:23.797",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573280493600773, '"checkpoint_time":' '"2024-05-06' '16:59:23.797",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7057.out cli changefeed pause -c changefeed-initialize-error =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 46-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449573280756006917, "checkpoint_time": "2024-05-06 16:59:24.798", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449573280756006917, "checkpoint_time": "2024-05-06 16:59:24.798", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449573280756006917, "checkpoint_time": "2024-05-06 16:59:24.798", "error": { "time": "2024-05-06T16:59:17.049734224+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449573280756006917, '"checkpoint_time":' '"2024-05-06' '16:59:24.798",' '"error":' '{' '"time":' '"2024-05-06T16:59:17.049734224+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449573280756006917, '"checkpoint_time":' '"2024-05-06' '16:59:24.798",' '"error":' '{' '"time":' '"2024-05-06T16:59:17.049734224+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7150.out cli changefeed resume -c changefeed-initialize-error check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 47-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh: line 1: 16907 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 (wd: /tmp/tidb_cdc_test/kafka_column_selector) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 + info='{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573280756006917, "checkpoint_time": "2024-05-06 16:59:24.798", "error": null }' + echo '{ "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573280756006917, "checkpoint_time": "2024-05-06 16:59:24.798", "error": null }' { "upstream_id": 7365808283075265434, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449573280756006917, "checkpoint_time": "2024-05-06 16:59:24.798", "error": null } ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573280756006917, '"checkpoint_time":' '"2024-05-06' '16:59:24.798",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365808283075265434, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449573280756006917, '"checkpoint_time":' '"2024-05-06' '16:59:24.798",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7228.out cli changefeed remove -c changefeed-initialize-error VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484786c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:7584, start at 2024-05-06 16:59:27.180246877 +0800 CST m=+6.045898619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:27.201 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:27.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:27.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3484786c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:7584, start at 2024-05-06 16:59:27.180246877 +0800 CST m=+6.045898619 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:27.201 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:27.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:27.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348479740003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:7667, start at 2024-05-06 16:59:27.199285108 +0800 CST m=+6.004796901 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:27.207 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:27.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:27.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449573280756006917 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-1327?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 table ddl_manager.finish_mark not exists for 48-th check, retry later + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.9047.out cli tso query --pd=http://127.0.0.1:2379 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 49-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:32 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 354 369 --:--:-- --:--:-- --:--:-- 373 {"compatibility":"NONE"}The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 16:59:33 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x + tso='449573282612772866 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573282612772866 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff failed 1-th time, retry later Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 50-th check, retry later ***************** properties ***************** "insertproportion"="0" "workload"="core" "readallfields"="true" "mysql.user"="root" "mysql.port"="4000" "threadcount"="2" "updateproportion"="0" "dotransactions"="false" "scanproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "mysql.db"="multi_capture_1" "recordcount"="10" "readproportion"="0" ********************************************** Run finished, takes 11.780077ms INSERT - Takes(s): 0.0, Count: 9, OPS: 1465.1, Avg(us): 1927, Min(us): 1326, Max(us): 5531, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "operationcount"="0" "mysql.user"="root" "mysql.port"="4000" "dotransactions"="false" "threadcount"="2" "mysql.host"="127.0.0.1" "readproportion"="0" "scanproportion"="0" "recordcount"="10" "updateproportion"="0" "mysql.db"="multi_capture_2" "requestdistribution"="uniform" "workload"="core" "readallfields"="true" "insertproportion"="0" ********************************************** Run finished, takes 20.892777ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1269.9, Avg(us): 4090, Min(us): 1418, Max(us): 13000, 95th(us): 13000, 99th(us): 13000 check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 51-th check, retry later ***************** properties ***************** "recordcount"="10" "operationcount"="0" "mysql.db"="multi_capture_3" "readproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "insertproportion"="0" "mysql.port"="4000" "workload"="core" "scanproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" ********************************************** start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector_avro Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Run finished, takes 24.612269ms INSERT - Takes(s): 0.0, Count: 10, OPS: 708.1, Avg(us): 4638, Min(us): 2245, Max(us): 10805, 95th(us): 11000, 99th(us): 11000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "requestdistribution"="uniform" "recordcount"="10" "scanproportion"="0" "readproportion"="0" "insertproportion"="0" "readallfields"="true" "mysql.db"="multi_capture_4" "mysql.host"="127.0.0.1" "dotransactions"="false" "updateproportion"="0" "operationcount"="0" "mysql.user"="root" "threadcount"="2" "workload"="core" "mysql.port"="4000" ********************************************** Run finished, takes 17.911241ms INSERT - Takes(s): 0.0, Count: 9, OPS: 765.7, Avg(us): 3127, Min(us): 2328, Max(us): 6088, 95th(us): 7000, 99th(us): 7000 [Mon May 6 16:59:37 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.91829184.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 52-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348531600013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:6838, start at 2024-05-06 16:59:38.991183434 +0800 CST m=+5.081299225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:38.997 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:38.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:38.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348531600013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:6838, start at 2024-05-06 16:59:38.991183434 +0800 CST m=+5.081299225 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:38.997 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:38.968 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:38.968 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34853450000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:6919, start at 2024-05-06 16:59:39.172434328 +0800 CST m=+5.212663861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:39.179 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:39.156 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:39.156 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 53-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 16:59:37.401 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/06 16:59:37.401 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/06 16:59:37.924 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs58ddcd4c_5b61_4b36_8c94_8571f9480f57"] [2024/05/06 16:59:37.931 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbc96f063_f61b_4583_9a17_58a00b5fe4b9"] [2024/05/06 16:59:37.932 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs883bda63_3db8_4ca5_9d56_fa2620224757"] [2024/05/06 16:59:37.933 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbcc26685_0f73_4056_a39c_027f6ba8b0ba"] [2024/05/06 16:59:37.935 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse4f8b948_3637_400c_9538_f8829d3d0142"] [2024/05/06 16:59:37.936 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa7ca5d73_1e57_4c43_9ee0_bd9bab3463cf"] [2024/05/06 16:59:37.937 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscf339b2e_6dee_4874_b24a_ac2f896d284c"] [2024/05/06 16:59:37.938 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs193600d2_fd3c_443b_97e2_f34f8dd02885"] [2024/05/06 16:59:37.961 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 16:59:37.961 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 16:59:38.042 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 16:59:38.042 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 16:59:38.449 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.450 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.451 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.454 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.515 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.516 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.516 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.518 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.519 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:38.521 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.525 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.527 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.528 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.528 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.528 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.541 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 16:59:38.544 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 16:59:38.576 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:38.727 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 16:59:38.731 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 16:59:38.740 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 16:59:38.744 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 16:59:38.924 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:38.925 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:38.929 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:38.931 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:39.041 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:39.045 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:39.046 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:39.050 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:39.056 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.056 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.114 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.116 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.118 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.121 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.123 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.126 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 16:59:39.131 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:39.139 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 16:59:39.425 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.426 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.431 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.432 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.434 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 16:59:39.533 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 16:59:39.541 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.550 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.552 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.553 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:39.636 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.637 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.716 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.716 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.719 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.724 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.725 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 16:59:39.729 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.730 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:39.756 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 16:59:39.931 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:39.933 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:39.940 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:39.941 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:40.117 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:40.126 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:40.126 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:40.131 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:40.153 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 16:59:40.158 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 16:59:40.214 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.215 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.252 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.254 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.259 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.262 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.264 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.265 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 16:59:40.316 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:40.340 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 16:59:40.345 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 16:59:40.357 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 16:59:40.528 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.528 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.536 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.542 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.658 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.720 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.722 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.726 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:40.749 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.753 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.844 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.845 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.850 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.850 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.852 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.857 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:40.860 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 16:59:40.945 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 16:59:40.968 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 16:59:41.058 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:41.115 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:41.122 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:41.135 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:40 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 16:59:41 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.92379239.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 16:59:41.227 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 16:59:41.238 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:41.254 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:41.256 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:41.257 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:41.332 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.345 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.427 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.428 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.433 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.434 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.434 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.439 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:41.445 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 16:59:41.559 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 16:59:41.644 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:41.645 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:41.655 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.finish_mark not exists for 4-th check, retry later [2024/05/06 16:59:41.723 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:41.733 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 16:59:41.737 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 16:59:41.819 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:41.835 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:41.837 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:41.839 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:41.848 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:41.862 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:41.949 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:41.951 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:41.955 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:41.956 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:41.960 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:41.962 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:42.021 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 16:59:42.056 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 16:59:42.060 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 16:59:42.136 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] check diff failed 5-th time, retry later [Mon May 6 16:59:42 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.82488250.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 16:59:42.220 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.223 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.235 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.252 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.346 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.414 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:42.419 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.419 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.422 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:42.433 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 5-th check, retry later [2024/05/06 16:59:42.524 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 16:59:42.525 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:42.528 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:42.533 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:42.536 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:42.545 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:42.551 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:42.556 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] table ddl_manager.finish_mark not exists for 54-th check, retry later [2024/05/06 16:59:42.731 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 16:59:42.823 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:42.829 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:42.837 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:42.857 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:42.937 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:42.944 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 16:59:42.956 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:42.960 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:43.015 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:43.019 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:43.026 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:43.057 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:43.121 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:43.124 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:43.128 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:43.138 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:43.144 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 16:59:43.144 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:43.320 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 16:59:43.325 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 16:59:43.332 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 16:59:43.355 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.359 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.422 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.440 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.517 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.537 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.541 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.546 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.549 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:43.552 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.636 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.652 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.654 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.715 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.724 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.724 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:43.726 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 16:59:43.749 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 16:59:43.754 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 16:59:43.854 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 16:59:43.926 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:43.934 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:43.946 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:44.016 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:44.042 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:44.050 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/06 16:59:44.058 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:44.119 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:44.123 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:44.127 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:44.135 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:44.166 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 5-th check, retry later [2024/05/06 16:59:44.227 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:44.229 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:44.244 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:44.254 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 16:59:44.255 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:44.260 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:44.447 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 16:59:44.451 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:44 GMT < Content-Length: 1271 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/capture/bbf82d3c-6e78-4510-9abd-5fab66f1d496 {"id":"bbf82d3c-6e78-4510-9abd-5fab66f1d496","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed28e bbf82d3c-6e78-4510-9abd-5fab66f1d496 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/capture/bbf82d3c-6e78-4510-9abd-5fab66f1d496 {"id":"bbf82d3c-6e78-4510-9abd-5fab66f1d496","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed28e bbf82d3c-6e78-4510-9abd-5fab66f1d496 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/capture/bbf82d3c-6e78-4510-9abd-5fab66f1d496 {"id":"bbf82d3c-6e78-4510-9abd-5fab66f1d496","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed28e bbf82d3c-6e78-4510-9abd-5fab66f1d496 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 16:59:44 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.92869288.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 6-th check, retry later [2024/05/06 16:59:44.518 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:44.534 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:44.551 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:44.621 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/06 16:59:44.634 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:44.654 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:44.725 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:44.729 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:44.729 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] table ddl_manager.finish_mark not exists for 55-th check, retry later check diff successfully [2024/05/06 16:59:44.747 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:44.825 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:44.842 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:44.847 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:44.851 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 16:59:44.854 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:44.856 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/06 16:59:44.917 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 16:59:44.922 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:44.923 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 16:59:45.046 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.118 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 16:59:45.121 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.138 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.152 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.230 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.247 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:45.325 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.326 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.332 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:45.340 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:45.413 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:45.436 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:45.439 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:45.441 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:45.449 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 16:59:45.454 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/06 16:59:45.460 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 16:59:45.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:45 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/db6afed8-d8dd-448a-87b7-b674b5c6f0fe {"id":"db6afed8-d8dd-448a-87b7-b674b5c6f0fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985982} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2120c4c7 db6afed8-d8dd-448a-87b7-b674b5c6f0fe /tidb/cdc/default/default/upstream/7365808655107577901 {"id":7365808655107577901,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/db6afed8-d8dd-448a-87b7-b674b5c6f0fe {"id":"db6afed8-d8dd-448a-87b7-b674b5c6f0fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985982} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2120c4c7 db6afed8-d8dd-448a-87b7-b674b5c6f0fe /tidb/cdc/default/default/upstream/7365808655107577901 {"id":7365808655107577901,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/db6afed8-d8dd-448a-87b7-b674b5c6f0fe {"id":"db6afed8-d8dd-448a-87b7-b674b5c6f0fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985982} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d2120c4c7 db6afed8-d8dd-448a-87b7-b674b5c6f0fe /tidb/cdc/default/default/upstream/7365808655107577901 {"id":7365808655107577901,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 16:59:45 CST 2024] <<<<<< START kafka consumer in processor_stop_delay case >>>>>> [2024/05/06 16:59:45.515 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:45.621 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:45.645 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:45.653 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 16:59:45.713 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 16:59:45.724 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:45.738 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:45.756 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table processor_stop_delay.t not exists for 1-th check, retry later [2024/05/06 16:59:45.821 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:45.858 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:45.915 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:45.920 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:45.922 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:45.942 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:46.020 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:46.024 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:46.026 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:46.047 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/06 16:59:46.049 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:46.120 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:46.226 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later [2024/05/06 16:59:46.318 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:46.340 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/06 16:59:46.350 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:46.416 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 16:59:46.424 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:46.432 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:46.445 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] table ddl_manager.finish_mark not exists for 56-th check, retry later [2024/05/06 16:59:46.542 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:46.546 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:46.547 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 16:59:46.551 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:46.552 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 16:59:46.554 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:46.616 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:46.642 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:46.643 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:46.645 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:46.663 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 16:59:46.715 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:46.726 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 16:59:46.823 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:46.915 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:46.941 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 16:59:46.943 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:47.026 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:47.030 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:47.040 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.146 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:47.148 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.155 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:47.214 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:47.229 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.248 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.249 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.251 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 16:59:47.255 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 16:59:47.256 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.327 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 16:59:47.332 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.340 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:47.365 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/06 16:59:47.421 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:47.457 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:47.537 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:47.547 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 16:59:47.615 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:47.727 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:47.749 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:47.753 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:47.764 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:47 GMT < Content-Length: 1750 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17054655-7a7f-422c-98bb-33a3aef8ff53 {"id":"17054655-7a7f-422c-98bb-33a3aef8ff53","address":"127.0.0.1:8303","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985984} /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/capture/bbf82d3c-6e78-4510-9abd-5fab66f1d496 {"id":"bbf82d3c-6e78-4510-9abd-5fab66f1d496","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed28e bbf82d3c-6e78-4510-9abd-5fab66f1d496 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed2aa 17054655-7a7f-422c-98bb-33a3aef8ff53 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17054655-7a7f-422c-98bb-33a3aef8ff53 {"id":"17054655-7a7f-422c-98bb-33a3aef8ff53","address":"127.0.0.1:8303","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985984} /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/capture/bbf82d3c-6e78-4510-9abd-5fab66f1d496 {"id":"bbf82d3c-6e78-4510-9abd-5fab66f1d496","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed28e bbf82d3c-6e78-4510-9abd-5fab66f1d496 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed2aa 17054655-7a7f-422c-98bb-33a3aef8ff53 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/17054655-7a7f-422c-98bb-33a3aef8ff53 {"id":"17054655-7a7f-422c-98bb-33a3aef8ff53","address":"127.0.0.1:8303","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985984} /tidb/cdc/default/__cdc_meta__/capture/9e0599e1-5790-49e7-bd68-53459da0c8a4 {"id":"9e0599e1-5790-49e7-bd68-53459da0c8a4","address":"127.0.0.1:8301","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985978} /tidb/cdc/default/__cdc_meta__/capture/bbf82d3c-6e78-4510-9abd-5fab66f1d496 {"id":"bbf82d3c-6e78-4510-9abd-5fab66f1d496","address":"127.0.0.1:8302","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed255 9e0599e1-5790-49e7-bd68-53459da0c8a4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed28e bbf82d3c-6e78-4510-9abd-5fab66f1d496 /tidb/cdc/default/__cdc_meta__/owner/22318f4d20eed2aa 17054655-7a7f-422c-98bb-33a3aef8ff53 /tidb/cdc/default/default/upstream/7365808604211154684 {"id":7365808604211154684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.9339.out cli changefeed create --start-ts=449573282612772866 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-capture-test-17843?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8301 table processor_stop_delay.t not exists for 2-th check, retry later [2024/05/06 16:59:47.831 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:47.837 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:47.843 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:47.860 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/06 16:59:47.866 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:47.873 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:47.965 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 16:59:48.038 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 16:59:48.039 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] Create changefeed successfully! ID: 68c2ea5b-1640-4e45-a64c-7dca71228181 Info: {"upstream_id":7365808604211154684,"namespace":"default","id":"68c2ea5b-1640-4e45-a64c-7dca71228181","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-capture-test-17843?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T16:59:47.906280227+08:00","start_ts":449573282612772866,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573282612772866,"checkpoint_ts":449573282612772866,"checkpoint_time":"2024-05-06 16:59:31.881"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 7-th check, retry later [2024/05/06 16:59:48.057 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/06 16:59:48.062 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 16:59:48.121 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/06 16:59:48.125 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/05/06 16:59:48.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 16:59:48.244 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 16:59:48.262 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] table test.finish_mark not exists for 8-th check, retry later [2024/05/06 16:59:48.316 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:48.324 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:48.334 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/06 16:59:48.339 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:48.387 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 16:59:48.469 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 16:59:48.545 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfc92d2c5_7e87_47e3_b5b8_bbdbb16d6091"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485bc280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:29350, start at 2024-05-06 16:59:47.873463288 +0800 CST m=+5.434569313 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:47.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:47.850 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:47.850 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 16:59:48.632 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 16:59:48.646 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [2024/05/06 16:59:48.648 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 16:59:48.671 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 16:59:48.744 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 16:59:48.752 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 16:59:48.761 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/06 16:59:48.778 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] check diff failed 1-th time, retry later [2024/05/06 16:59:48.827 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/05/06 16:59:48.923 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/05/06 16:59:48.963 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:48.963 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:49.032 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/05/06 16:59:49.057 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs63e8c271_9e0e_429a_873c_8570aaecd8ac"] table ddl_manager.finish_mark not exists for 57-th check, retry later + set +x [Mon May 6 16:59:49 CST 2024] <<<<<< START kafka consumer in multi_capture case >>>>>> [2024/05/06 16:59:49.118 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1945d358_0a2e_48d6_8b6f_e7cb19ec40fd"] [2024/05/06 16:59:49.130 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/05/06 16:59:49.137 +08:00] [INFO] [main.go:178] ["0 insert success: 1600"] [2024/05/06 16:59:49.141 +08:00] [INFO] [main.go:199] ["0 delete success: 800"] [2024/05/06 16:59:49.153 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse4867aed_2c81_4ae3_b717_85d3b3d36eb1"] [2024/05/06 16:59:49.255 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:49.266 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:49.373 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:49.378 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:49.451 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:49.452 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:49.464 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/05/06 16:59:49.483 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:49.494 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:49.558 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:49.563 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsaddbec0a_aff9_486d_98db_6d10a0884d91"] [2024/05/06 16:59:49.568 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs78154f26_7cef_4bf8_9415_6c17e31335aa"] [2024/05/06 16:59:49.574 +08:00] [INFO] [main.go:178] ["0 insert success: 1700"] [2024/05/06 16:59:49.576 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6afa19f8_b4eb_4a73_9a51_70d149a44df0"] [2024/05/06 16:59:49.581 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table multi_capture_1.usertable not exists for 1-th check, retry later [2024/05/06 16:59:49.647 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3859973c_d86e_4670_abfe_2bbe7a23dfbf"] [2024/05/06 16:59:49.665 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:49.674 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:49.841 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:49.847 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:49.856 +08:00] [INFO] [main.go:178] ["1 insert success: 2300"] table test.finish_mark not exists for 8-th check, retry later [2024/05/06 16:59:49.929 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:49.938 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:50.056 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:50.063 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:50.113 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:50.113 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:50.116 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:50.120 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:50.127 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:50.130 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table test.finish_mark not exists for 9-th check, retry later table processor_stop_delay.t exists check diff successfully [2024/05/06 16:59:50.156 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:50.164 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:50.174 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:50.179 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:50.255 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:50.260 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:50.337 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:50.342 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485bc280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:29350, start at 2024-05-06 16:59:47.873463288 +0800 CST m=+5.434569313 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:47.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:47.850 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:47.850 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3485bdc00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:29421, start at 2024-05-06 16:59:47.981989805 +0800 CST m=+5.487155508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:01:47.989 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-16:59:47.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:49:47.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/06 16:59:50.460 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:50.554 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:50.558 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:50.562 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:50.565 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:50.570 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:50.620 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:50.622 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:50.634 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:50.641 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:50.662 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:50.666 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 16:59:50.695 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:50.723 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:50.748 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:50.756 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:50.856 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later [2024/05/06 16:59:50.954 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:51.015 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:51.017 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:51.022 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:51.030 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:51.060 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:51.068 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:51.076 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:51.122 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:51.135 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:51.138 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:51.162 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:51.166 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:51.220 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:51.222 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:51.276 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:51.358 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:51.431 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:51.434 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:51.439 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:51.443 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table multi_capture_1.usertable not exists for 2-th check, retry later [2024/05/06 16:59:51.475 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:51.522 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:51.546 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:51.561 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:51.565 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:51.616 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:51.642 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:51.648 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:51.664 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:51.671 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 9-th check, retry later [2024/05/06 16:59:51.745 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:51.824 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:51.852 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:51.856 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:51.860 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:51.865 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:51.933 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:51.943 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:51.960 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table test.finish_mark not exists for 10-th check, retry later [2024/05/06 16:59:51.972 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:52.027 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:52.031 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 16:59:52.054 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:52.060 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:52.083 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:52.083 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:52.128 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 14.726838094s"] [2024/05/06 16:59:52.155 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:52.221 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:52.248 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:52.255 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:52.263 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:52.315 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:52.348 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:52.362 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:52.416 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 16:59:52.433 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:52.464 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:52.465 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 16:59:52.526 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:52.545 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:52.554 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:52.564 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:52.634 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:52.662 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:52.729 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:52.731 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 59-th check, retry later [2024/05/06 16:59:52.751 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:52.753 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:52.821 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:52.834 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 16:59:52.842 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:52.853 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:52.930 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 16:59:52.930 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:52.942 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:52.966 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:52.971 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:53.023 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:53.045 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:53.072 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:53.135 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:53.138 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 16:59:53.162 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:53.163 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:53.227 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:53.230 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [Mon May 6 16:59:52 CST 2024] <<<<<< START cdc server in kafka_column_selector_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.3077730779.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 16:59:53.245 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:53.249 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:53.326 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 16:59:53.329 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:53.331 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:53.360 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:53.361 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:53.383 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:53.428 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:53.461 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later [2024/05/06 16:59:53.524 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:53.533 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:53.561 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:53.561 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:53.630 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 16:59:53.641 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:53.650 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:53.658 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:53.743 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:53.751 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 16:59:53.073 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 16:59:53.073 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365808220801092564] [2024/05/06 16:59:53.074 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 16:59:53.074 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 16:59:53.074 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 16:59:53.074 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 16:59:53.075 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 16:59:53.075 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365808220801092564] [2024/05/06 16:59:53.076 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 16:59:53.076 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 16:59:53.076 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 16:59:53.076 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 16:59:53.076 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/06 16:59:53.077 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/06 16:59:53.079 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=7] [addr=127.0.0.1:20160] [from=unresolved] [to=resolved] [liveness-state=reachable] [2024/05/06 16:59:53.752 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:53.775 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:53.820 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:53.836 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:53.838 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:53.867 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:53.928 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:53.932 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff failed 2-th time, retry later [2024/05/06 16:59:54.070 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:54.070 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:54.090 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 16:59:54.108 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:54.119 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:54.136 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:54.216 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:54.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:54.234 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 16:59:54.254 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] table test.finish_mark not exists for 10-th check, retry later [2024/05/06 16:59:54.328 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:54.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:54.344 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:54.358 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:54.358 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:54.358 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 11-th check, retry later [2024/05/06 16:59:54.628 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 16:59:54.632 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:54.635 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:54.635 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:54.648 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 16:59:54.714 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:54.739 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:54.767 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:54.771 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] table ddl_manager.finish_mark not exists for 60-th check, retry later [2024/05/06 16:59:54.815 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:54.848 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:54.857 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:54.873 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:54.874 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:54.874 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:54.876 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 16:59:55.039 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:55.040 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 16:59:55.055 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:55.055 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:55.065 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:55.120 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:55.136 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:55.217 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:55.220 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:55.222 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 16:59:55.256 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] check diff failed 4-th time, retry later [2024/05/06 16:59:55.313 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:55.315 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:55.319 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:55.327 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:55.329 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:55.453 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:55.454 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 16:59:55.469 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:55.516 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:55.522 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:55.554 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:55.560 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:55.627 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:55.643 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 16:59:55.643 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:55.752 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:55.754 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:55.758 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:55.759 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/05/06 16:59:55.821 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:55.914 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:55.914 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 16:59:55.958 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:55.962 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:55.969 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:56.021 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later check diff failed 3-th time, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 08:59:55 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb4b0a30-b08d-4c0d-b054-0916e8c4750f {"id":"cb4b0a30-b08d-4c0d-b054-0916e8c4750f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985993} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21421ace cb4b0a30-b08d-4c0d-b054-0916e8c4750f /tidb/cdc/default/default/upstream/7365808695873242661 {"id":7365808695873242661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb4b0a30-b08d-4c0d-b054-0916e8c4750f {"id":"cb4b0a30-b08d-4c0d-b054-0916e8c4750f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985993} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21421ace cb4b0a30-b08d-4c0d-b054-0916e8c4750f /tidb/cdc/default/default/upstream/7365808695873242661 {"id":7365808695873242661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb4b0a30-b08d-4c0d-b054-0916e8c4750f {"id":"cb4b0a30-b08d-4c0d-b054-0916e8c4750f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714985993} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21421ace cb4b0a30-b08d-4c0d-b054-0916e8c4750f /tidb/cdc/default/default/upstream/7365808695873242661 {"id":7365808695873242661,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.cli.30831.out cli changefeed create --start-ts=449573288109670405 '--sink-uri=kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 [2024/05/06 16:59:56.044 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:56.081 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:56.085 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 16:59:56.146 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:56.149 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:56.159 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 16:59:56.186 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:56.227 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:56.268 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:56.269 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] table test.finish_mark not exists for 11-th check, retry later Create changefeed successfully! ID: test Info: {"upstream_id":7365808695873242661,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true","create_time":"2024-05-06T16:59:56.454260085+08:00","start_ts":449573288109670405,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.*"],"columns":["*","!b"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573288109670405,"checkpoint_ts":449573288109670405,"checkpoint_time":"2024-05-06 16:59:52.850"} PASS [2024/05/06 16:59:56.413 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:56.413 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:56.424 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 16:59:56.441 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] table test.finish_mark not exists for 12-th check, retry later table ddl_manager.finish_mark not exists for 61-th check, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/06 16:59:56.624 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:56.756 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] Verifying downstream PD is started... [2024/05/06 16:59:56.841 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:56.852 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:56.947 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:57.015 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:57.038 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 16:59:57.069 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:57.069 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:57.072 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:57.118 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:57.184 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 16:59:57.204 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:57.217 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:57.258 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] check diff failed 5-th time, retry later [2024/05/06 16:59:57.350 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 16:59:57.372 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:57.372 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:57.391 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 16:59:57.499 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 16:59:57.523 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:57.535 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:57.577 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 16:59:57.647 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 16:59:57.670 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa2a6d761_d762_4631_b85a_607e576ced77"] [2024/05/06 16:59:57.670 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 16:59:57.800 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table multi_capture_4.usertable exists check diff failed 1-th time, retry later check diff failed 4-th time, retry later + set +x [Mon May 6 16:59:57 CST 2024] <<<<<< START kafka consumer in kafka_column_selector_avro case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml schema registry uri found: http://127.0.0.1:8088 [2024/05/06 16:59:57.834 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 16:59:57.958 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 16:59:57.979 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 16:59:57.982 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:57.983 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:58.104 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 16:59:58.141 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/05/06 16:59:58.172 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9e1d1503_c4ad_4f5d_9875_c3c10160ef15"] [2024/05/06 16:59:58.190 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf22901db_ac5b_4249_8e59_8442e4c87dc0"] [2024/05/06 16:59:58.236 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0f29c765_a0ef_4b3b_8b6d_f3eb969289a8"] [2024/05/06 16:59:58.243 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7f52fdac_e314_4b7c_9bac_a284a9f15121"] [2024/05/06 16:59:58.253 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs19ae70f1_cfb7_4d66_8b78_30709d99bc5c"] [2024/05/06 16:59:58.283 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/05/06 16:59:58.290 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:58.295 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table test.finish_mark not exists for 13-th check, retry later Starting build checksum checker... table test.finishmark not exists for 1-th check, retry later [2024/05/06 16:59:58.630 +08:00] [INFO] [main.go:178] ["73 insert success: 2100"] [2024/05/06 16:59:58.736 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:58.736 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:58.753 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:58.847 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:58.848 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:58.851 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:58.857 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:58.859 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:58.872 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:58.884 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:58.884 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:58.893 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:59.052 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:59.057 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:59.069 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table ddl_manager.finish_mark not exists for 62-th check, retry later check diff successfully [2024/05/06 16:59:59.268 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdf23e949_c5ea_49a3_9859_bb04855aef0f"] [2024/05/06 16:59:59.369 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:59.373 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:59.375 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:59.414 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:59.415 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:59.430 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 16:59:59.438 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 16:59:59.442 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 16:59:59.450 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 16:59:59.535 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:59.538 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:59.546 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ***************** properties ***************** "insertproportion"="0" "workload"="core" "recordcount"="20" "scanproportion"="0" "mysql.db"="multi_capture_1" "dotransactions"="false" "updateproportion"="0" "readproportion"="0" "mysql.user"="root" "threadcount"="2" "mysql.host"="127.0.0.1" "operationcount"="0" "readallfields"="true" "requestdistribution"="uniform" "mysql.port"="4000" ********************************************** Run finished, takes 14.642168ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1584.0, Avg(us): 1390, Min(us): 758, Max(us): 2038, 95th(us): 2000, 99th(us): 3000 [2024/05/06 16:59:59.724 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 16:59:59.845 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 16:59:59.847 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:59.849 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:59.849 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ***************** properties ***************** "workload"="core" "mysql.host"="127.0.0.1" "readallfields"="true" "scanproportion"="0" "mysql.db"="multi_capture_2" "dotransactions"="false" "updateproportion"="0" "recordcount"="20" "mysql.port"="4000" "readproportion"="0" "mysql.user"="root" "operationcount"="0" "insertproportion"="0" "requestdistribution"="uniform" "threadcount"="2" ********************************************** Run finished, takes 18.939918ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1353.3, Avg(us): 1796, Min(us): 664, Max(us): 5366, 95th(us): 5000, 99th(us): 6000 ***************** properties ***************** "scanproportion"="0" "mysql.user"="root" "readproportion"="0" "insertproportion"="0" "workload"="core" "readallfields"="true" "mysql.db"="multi_capture_3" "threadcount"="2" "dotransactions"="false" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "operationcount"="0" "mysql.port"="4000" "updateproportion"="0" "recordcount"="20" ********************************************** Run finished, takes 12.724209ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1774.3, Avg(us): 1103, Min(us): 598, Max(us): 1861, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "recordcount"="20" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_4" "readallfields"="true" "mysql.user"="root" "workload"="core" "readproportion"="0" "insertproportion"="0" "mysql.port"="4000" "threadcount"="2" "dotransactions"="false" "requestdistribution"="uniform" "scanproportion"="0" "operationcount"="0" "updateproportion"="0" ********************************************** Run finished, takes 27.044056ms INSERT - Takes(s): 0.0, Count: 20, OPS: 928.5, Avg(us): 2626, Min(us): 1614, Max(us): 7555, 95th(us): 6000, 99th(us): 8000 [2024/05/06 16:59:59.943 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:59.943 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:59.957 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 16:59:59.958 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 16:59:59.969 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 16:59:59.972 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:00.055 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:00.062 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:00.066 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:00.157 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff failed 1-th time, retry later [2024/05/06 17:00:00.418 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:00.421 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table test.finish_mark not exists for 14-th check, retry later table test.finish_mark exists check diff successfully table test.finishmark not exists for 2-th check, retry later check diff failed 5-th time, retry later [2024/05/06 17:00:00.425 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:00.425 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:00.532 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:00.532 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:00.549 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:00.549 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:00.558 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:00.631 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:00.715 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:00.725 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:00.733 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:00.747 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsffdaa772_d713_466a_8bcf_683e618b521f"] [2024/05/06 17:00:00.835 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table ddl_manager.finish_mark not exists for 63-th check, retry later [2024/05/06 17:00:01.031 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:01.113 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:01.118 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:01.118 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:01.234 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:01.234 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:01.254 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:01.254 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:01.258 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:01.334 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:01.350 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:01.357 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:01.369 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:01.417 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:01.422 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:01.446 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:01.534 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:01.625 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:01.629 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:01.639 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:01.841 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:01.846 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:01.914 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:01.919 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:01.925 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:01.967 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:01.970 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:02.026 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:02.039 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:02.050 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:02.052 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:02.075 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:02.123 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:02.143 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:02.148 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table test.finish_mark not exists for 15-th check, retry later [2024/05/06 17:00:02.217 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:02.431 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff failed 6-th time, retry later [2024/05/06 17:00:02.522 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:02.539 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:02.543 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:02.617 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:02.661 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:02.663 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 64-th check, retry later [2024/05/06 17:00:02.731 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:02.742 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:02.756 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:02.757 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:02.762 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:02.765 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:02.827 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:02.829 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:02.843 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:02.938 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.finishmark exists [2024/05/06 17:00:02.451 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/06 17:00:02.453 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/06 17:00:02.454 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=8.717513ms] [2024/05/06 17:00:02.461 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/06 17:00:02.464 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/06 17:00:02.464 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=10.13498ms] [2024/05/06 17:00:02.464 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 1-th time... [2024/05/06 17:00:03.042 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:03.117 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:03.133 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later [2024/05/06 17:00:03.233 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:03.349 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:03.350 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:03.434 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:03.440 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] wait process cdc.test exit for 2-th time... [2024/05/06 17:00:03.515 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:03.516 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:03.518 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:03.527 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:03.549 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:03.551 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:03.617 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:03.646 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:03.725 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:03.087 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/06 17:00:03.087 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/06 17:00:03.087 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/06 17:00:03.087 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/06 17:00:03.087 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/06 17:00:03.088 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/06 17:00:03.088 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/06 17:00:03.088 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/06 17:00:03.088 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 17:00:03.088 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/06 17:00:03.088 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 17:00:03.088 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/06 17:00:03.088 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] wait process cdc.test exit for 3-th time... [2024/05/06 17:00:03.734 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:03.743 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:03.925 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 17:00:04 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 16-th check, retry later [2024/05/06 17:00:04.046 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:04.121 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:04.216 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:04.226 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:04.238 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] check diff successfully cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:04 CST 2024] <<<<<< run test case multi_capture success! >>>>>> [2024/05/06 17:00:04.313 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:04.321 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:04.332 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:04.348 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:04.353 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:04.417 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:04.525 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:04.535 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:04.536 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:04.623 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:04.659 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table ddl_manager.finish_mark not exists for 65-th check, retry later wait process cdc.test exit for 1-th time... [2024/05/06 17:00:04.827 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:04.833 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:04.844 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:04.946 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] wait process cdc.test exit for 2-th time... [2024/05/06 17:00:05.018 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:05.033 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:05.034 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:05.043 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:05.120 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:05.224 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:05.228 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:05.243 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] check diff failed 2-th time, retry later [2024/05/06 17:00:05.269 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:05.275 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:05.359 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:05.363 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:05.373 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:05.461 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:05.468 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:05.472 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 17:00:05 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> [2024/05/06 17:00:05.518 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:05.532 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:05.633 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:05.820 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:05.843 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:05.858 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:05.923 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:05.941 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:05.974 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:06.016 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 17:00:06.022 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:06.056 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:06.068 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:06.068 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:06.075 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:06.119 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:06.158 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:06.328 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:06.436 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table test.finish_mark exists check diff successfully [2024/05/06 17:00:06.531 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:06.532 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:06.636 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:06.714 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:06.719 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:06.725 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] wait process cdc.test exit for 1-th time... [2024/05/06 17:00:06.815 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:06.827 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:06.851 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:06.916 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1bf5b266_7af9_401e_88b8_e31e7a293201"] [2024/05/06 17:00:06.920 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:06.947 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:07.017 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:07.116 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:07.152 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] table ddl_manager.finish_mark not exists for 66-th check, retry later [2024/05/06 17:00:07.318 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:07.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:07.327 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:07.336 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:07.422 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:07.433 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] wait process cdc.test exit for 2-th time... [2024/05/06 17:00:07.551 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:07.557 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:07.559 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:07.561 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:07.642 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:07.652 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:07.720 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:07.751 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:07.771 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:07.781 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 17:00:07.819 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486c7d80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:8281, start at 2024-05-06 17:00:04.986166746 +0800 CST m=+5.275130178 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:04.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:04.982 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:04.982 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486c7d80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:8281, start at 2024-05-06 17:00:04.986166746 +0800 CST m=+5.275130178 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:04.992 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:04.982 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:04.982 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3486c9580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-j10vp-2nv1x, pid:8356, start at 2024-05-06 17:00:05.105563029 +0800 CST m=+5.341467706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:05.112 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:05.078 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:05.078 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/06 17:00:07.831 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:07.943 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:08.024 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:08.027 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] wait process cdc.test exit for 3-th time... Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 17:00:08.121 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:08.160 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 17:00:08.163 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 17:00:08.262 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 17:00:08.345 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs26afcdbc_3bfb_49ee_ae20_5929b27345c1"] [2024/05/06 17:00:08.350 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 17:00:08.364 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:08.422 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 17:00:08.432 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:08.467 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:08.492 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:08.497 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:08.539 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:08 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/05/06 17:00:08.649 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:08.649 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:08.767 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:08.840 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] table test.t2 not exists for 1-th check, retry later [2024/05/06 17:00:08.936 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:08.944 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs584c77d5_0a43_4067_a9f7_2d76cca76937"] [2024/05/06 17:00:09.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:09.029 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table ddl_manager.finish_mark not exists for 67-th check, retry later [2024/05/06 17:00:09.113 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 17:00:09.224 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:09.225 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:09.314 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:09.348 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 17:00:09.367 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:09.382 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:09.391 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:09.434 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7426b697_9e08_4a5d_86c6_8f1e1abe0c4b"] [2024/05/06 17:00:09.454 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:09.455 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:09.466 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs76ce4a60_da69_4e6b_ac51_b8e42548487e"] [2024/05/06 17:00:09.538 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscf6f3cd2_9f1b_4ee9_8bfb_47c84163ee42"] check diff failed 4-th time, retry later [2024/05/06 17:00:09.652 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:09.655 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:09.840 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9818.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 17:00:09.950 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:10.017 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:10.021 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:10.046 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:10.052 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:10.117 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:10.126 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:10.130 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:10.152 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:10.156 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:10.256 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:10.259 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:10.458 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:10.519 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:10.523 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:10.551 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:10.554 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:10.573 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:10.576 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:10.585 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:10.627 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:10.633 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:10.750 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:10.762 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table test.t2 not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 68-th check, retry later [2024/05/06 17:00:11.016 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb475b13a_9e4a_4b7f_b162_fded768bdd82"] [2024/05/06 17:00:11.049 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:11.133 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:11.139 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:11.153 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:11.163 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:11.164 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:11.165 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:11.172 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:11.214 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:11.215 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:11.271 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:11.279 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:11.449 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:11.518 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:11.550 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] check diff failed 5-th time, retry later [2024/05/06 17:00:11.654 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:11.724 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:11.738 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:11.743 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:11.744 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:11.754 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:11.756 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:11.819 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:11.828 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:11.852 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:11.857 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:11.957 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:12.016 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:12.023 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs08308283_911b_4e29_8698_255d9561eaf4"] [2024/05/06 17:00:12.051 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:12.317 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:12.332 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:12.415 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:12.416 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:12.416 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:12.424 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:12.426 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:12.468 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:12.470 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:12.537 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:12.547 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:12.622 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:12.622 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:12.628 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:12.636 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:12.646 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 1-th check, retry later table test.t2 not exists for 3-th check, retry later + set +x + tso='449573292692471809 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573292692471809 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 17:00:12 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.98589860.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 69-th check, retry later [2024/05/06 17:00:12.944 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:12.949 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:13.145 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:13.216 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:13.219 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:13.222 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.finish_mark not exists for 2-th check, retry later [2024/05/06 17:00:13.231 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:13.333 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:13.333 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:13.417 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:13.428 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:13.445 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:13.449 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:13.454 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:13.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] check diff successfully [2024/05/06 17:00:13.516 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:13.650 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:13.660 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:13.767 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:13.816 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:13.824 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:13.824 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:13.828 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:13.920 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:13.933 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:13.955 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:14.035 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:14.116 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:14.126 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:14.126 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:14.130 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:14.215 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:14.358 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:14.360 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.t2 not exists for 4-th check, retry later [2024/05/06 17:00:14.517 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:14.519 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:14.520 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:14.532 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:14.536 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:14.553 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:14.572 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:14.618 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:14.629 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:14.635 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:14.649 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:14.651 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:14.655 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:14.728 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table ddl_manager.finish_mark not exists for 70-th check, retry later [2024/05/06 17:00:14.859 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:14.859 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:15.123 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:15.123 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:15.127 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:15.145 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:15.147 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:15.222 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:15.229 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:15.255 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:15.265 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:15.265 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:15.317 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:15.353 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:15.448 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:15.454 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:15.637 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:15.638 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:15.715 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:15.723 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:15.724 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table test.finish_mark not exists for 3-th check, retry later [2024/05/06 17:00:15.821 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:15.822 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:15.926 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:15.928 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:15.937 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 17:00:15.938 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e67d80d-d316-4370-bd4d-4e6b36a01bab {"id":"3e67d80d-d316-4370-bd4d-4e6b36a01bab","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986013} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d218952da 3e67d80d-d316-4370-bd4d-4e6b36a01bab /tidb/cdc/default/default/upstream/7365808762039679297 {"id":7365808762039679297,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e67d80d-d316-4370-bd4d-4e6b36a01bab {"id":"3e67d80d-d316-4370-bd4d-4e6b36a01bab","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986013} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d218952da 3e67d80d-d316-4370-bd4d-4e6b36a01bab /tidb/cdc/default/default/upstream/7365808762039679297 {"id":7365808762039679297,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3e67d80d-d316-4370-bd4d-4e6b36a01bab {"id":"3e67d80d-d316-4370-bd4d-4e6b36a01bab","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986013} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d218952da 3e67d80d-d316-4370-bd4d-4e6b36a01bab /tidb/cdc/default/default/upstream/7365808762039679297 {"id":7365808762039679297,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9910.out cli changefeed create --start-ts=449573292692471809 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-27871?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/05/06 17:00:16.053 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:16.136 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:16.148 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:16.236 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:16.236 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:16.252 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:16.255 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: kafka... <<================= Create changefeed successfully! ID: b9c938c7-5dd9-472f-9227-2bb7d7e0605a Info: {"upstream_id":7365808762039679297,"namespace":"default","id":"b9c938c7-5dd9-472f-9227-2bb7d7e0605a","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-27871?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T17:00:16.225498964+08:00","start_ts":449573292692471809,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573292692471809,"checkpoint_ts":449573292692471809,"checkpoint_time":"2024-05-06 17:00:10.332"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 17:00:16.280 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:16.282 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:16.318 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:16.322 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:16.326 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:16.392 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:16.521 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:16.533 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] table test.t2 not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... [2024/05/06 17:00:16.661 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:16.661 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:16.725 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:16.730 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:16.753 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:16.756 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:16.769 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:16.774 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:16.774 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:16.822 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:16.887 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:16.930 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:17.029 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd212fee9_2b92_40e4_8f35_fa5f6d28be56"] [2024/05/06 17:00:17.222 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:17.223 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:17.274 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table ddl_manager.finish_mark not exists for 71-th check, retry later [2024/05/06 17:00:17.314 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:17.342 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:17.356 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:17.359 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:17.363 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:17.377 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:17.442 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:17.446 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:17.479 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:17.515 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:17.523 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] check diff failed 1-th time, retry later table test.finish_mark not exists for 4-th check, retry later + set +x [Mon May 6 17:00:17 CST 2024] <<<<<< START kafka consumer in ddl_sequence case >>>>>> [2024/05/06 17:00:17.630 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:17.743 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:17.758 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/06 17:00:17.822 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:17.828 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:17.838 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:17.849 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:17.934 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 17:00:17.947 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:17.969 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:17.977 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 17:00:18.016 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:18.066 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:18.154 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:18.238 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:18.252 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:18.252 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 17:00:18.331 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:18.334 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:18.427 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 17:00:18.446 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:18.461 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:18.473 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 17:00:18.487 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:18.549 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 17:00:18.582 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 17:00:18.621 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 17:00:18.636 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:18.636 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:18.715 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:18.774 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 17:00:18.827 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:18.839 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:18.857 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 17:00:18.880 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:19.045 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfbd3230b_1b83_4520_8f1b_712ce68a684d"] [2024/05/06 17:00:19.048 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:19.050 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] table test.t2 not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 72-th check, retry later [2024/05/06 17:00:19.133 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:19.206 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 17:00:19.215 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:19.226 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb4e4b564_5ebe_45ee_ae3e_3fb8bd831bf0"] [2024/05/06 17:00:19.236 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs54a6ac0e_2572_43d7_a72d_6d2422fcc706"] [2024/05/06 17:00:19.258 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed [2024/05/06 17:00:19.448 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 17:00:19.450 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:19.451 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 17:00:19.452 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table test.finish_mark not exists for 5-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1415 100 678 100 737 1148 1248 --:--:-- --:--:-- --:--:-- 1247 100 1415 100 678 100 737 1147 1247 --:--:-- --:--:-- --:--:-- 1247 HTTP/1.1 201 Created Date: Mon, 06 May 2024 09:00:19 GMT Location: http://localhost:8083/connectors/my-connector Content-Type: application/json Content-Length: 678 Server: Jetty(9.4.51.v20230217) {"name":"my-connector","config":{"connector.class":"io.debezium.connector.mysql.MySqlConnector","tasks.max":"1","database.hostname":"127.0.0.1","database.port":"3310","database.user":"debezium","database.password":"dbz","database.server.id":"184054","topic.prefix":"dbserver1","schema.history.internal.kafka.bootstrap.servers":"127.0.0.1:9092","schema.history.internal.kafka.topic":"schemahistory.test","transforms":"x","transforms.x.type":"org.apache.kafka.connect.transforms.RegexRouter","transforms.x.regex":"(.*)","transforms.x.replacement":"output_debezium","binary.handling.mode":"base64","decimal.handling.mode":"double","name":"my-connector"},"tasks":[],"type":"source"}The 1 times to try to start tidb cluster... [2024/05/06 17:00:19.614 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 17:00:19.659 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:19.663 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 17:00:19.672 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:19.677 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:19.678 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:19.682 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:19.733 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:19.795 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:19.795 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 17:00:19.801 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:19.806 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 17:00:20.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 17:00:20.064 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:20.075 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] check diff failed 2-th time, retry later [2024/05/06 17:00:20.115 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:20.118 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 17:00:20.133 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:20.133 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:20.161 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs11f97fa8_6b3d_4b47_b2f3_d2984394054d"] [2024/05/06 17:00:20.227 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:20.257 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:20.267 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... [2024/05/06 17:00:20.469 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:20.475 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:20.480 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:20.526 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:20.527 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:20.545 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 17:00:20.572 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 17:00:20.582 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:20.624 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/06 17:00:20.648 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:20.846 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:20.852 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:20.855 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:20.886 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:20.889 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [2024/05/06 17:00:20.915 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 17:00:20.960 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 17:00:20.978 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:21.034 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:21.066 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] table ddl_manager.finish_mark exists check diff successfully table test.t2 not exists for 7-th check, retry later [2024/05/06 17:00:21.226 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:21.230 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:21.232 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:21.255 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 17:00:21.261 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:21.263 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:21.309 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 17:00:21.345 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:21.353 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:21.393 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] wait process cdc.test exit for 1-th time... table test.finish_mark exists check diff successfully [2024/05/06 17:00:21.642 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:21.646 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:21.663 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 17:00:21.670 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:21.676 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:21.715 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 17:00:21.760 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:21.781 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later [2024/05/06 17:00:21.943 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:21.947 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:21.959 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 17:00:21.978 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:21.985 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:22.017 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 17:00:22.058 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:22.070 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:22.233 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:22.237 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:22.248 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 17:00:22.275 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:22.284 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 17:00:22.315 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 17:00:22.344 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:22.366 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... [2024/05/06 17:00:22.513 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:22.517 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:22.529 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 17:00:22.573 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:22.582 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:22.617 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 17:00:22.627 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:22.795 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:22.802 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:22.814 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 17:00:22.872 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:22.881 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:22.918 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.t2 not exists for 8-th check, retry later wait process cdc.test exit for 4-th time... wait process cdc.test exit for 3-th time... [2024/05/06 17:00:23.093 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:23.099 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:23.112 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 17:00:23.172 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:23.180 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 17:00:23.223 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 17:00:23.402 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 17:00:23.416 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 17:00:23.475 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 17:00:23.523 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 17:00:23.690 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:23 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> [2024/05/06 17:00:23.702 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 17:00:23.775 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] start tidb cluster in /tmp/tidb_cdc_test/debezium Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 5-th time... check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_sequence.finish_mark not exists for 1-th check, retry later [2024/05/06 17:00:24.389 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v46.988731678s"] wait process cdc.test exit for 6-th time... table test.t2 not exists for 9-th check, retry later [2024/05/06 17:00:24.901 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 47.500156228s"] wait process cdc.test exit for 7-th time... table mark.finish_mark_1 not exists for 1-th check, retry later wait process cdc.test exit for 8-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Starting Upstream TiDB... table ddl_sequence.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 9-th time... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 10-th time... table test.t2 not exists for 10-th check, retry later cdc.test: no process found wait process cdc.test exit for 11-th time... process cdc.test already exit [Mon May 6 17:00:26 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> table mark.finish_mark_1 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table test.t2 not exists for 11-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 3-th check, retry later Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark exists Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table test.t2 not exists for 12-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 17:00:31 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34885a04001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:9712, start at 2024-05-06 17:00:30.764073454 +0800 CST m=+5.152842175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:30.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:30.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:30.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34885a04001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:9712, start at 2024-05-06 17:00:30.764073454 +0800 CST m=+5.152842175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:30.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:30.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:30.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d34885aa80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-cq1g3-bjvlr, pid:9789, start at 2024-05-06 17:00:30.787562104 +0800 CST m=+5.130604230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:30.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:30.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:30.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 exists check diff successfully wait process cdc.test exit for 1-th time... table mark.finish_mark_2 not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11218.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348875c00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:10718, start at 2024-05-06 17:00:32.540337674 +0800 CST m=+6.282629654 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:32.548 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:32.546 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:32.546 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348875c00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:10718, start at 2024-05-06 17:00:32.540337674 +0800 CST m=+6.282629654 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:32.548 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:32.546 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:32.546 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348877600006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-0v21p-pdxr4, pid:10781, start at 2024-05-06 17:00:32.609271313 +0800 CST m=+6.267182224 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:32.618 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:32.600 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:32.600 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 17:00:34 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> table mark.finish_mark_2 not exists for 3-th check, retry later + set +x + tso='449573298967937025 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573298967937025 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 17:00:35 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1125611258.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3488a4280017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb, pid:21242, start at 2024-05-06 17:00:35.5089479 +0800 CST m=+5.733064282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:35.516 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:35.516 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:35.516 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348890280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:32112, start at 2024-05-06 17:00:34.20875599 +0800 CST m=+5.708815765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:34.216 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:34.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:34.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348890280013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:32112, start at 2024-05-06 17:00:34.20875599 +0800 CST m=+5.708815765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:34.216 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:34.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:34.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d348891c40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-pp21w-gtjg6, pid:32168, start at 2024-05-06 17:00:34.319918476 +0800 CST m=+5.753862155 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:34.327 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:34.289 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:34.289 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/debezium/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/debezium/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/debezium/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.12084.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3488a4280017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb, pid:21242, start at 2024-05-06 17:00:35.5089479 +0800 CST m=+5.733064282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:35.516 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:35.516 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:35.516 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3488a42c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-mvw62-c62hb, pid:21323, start at 2024-05-06 17:00:35.501784978 +0800 CST m=+5.668305274 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:35.512 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:35.517 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:35.517 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later table mark.finish_mark_2 not exists for 4-th check, retry later [Mon May 6 17:00:38 CST 2024] <<<<<< START cdc server in debezium case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.3350033502.out server --log-file /tmp/tidb_cdc_test/debezium/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/debezium/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449573299708493825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573299708493825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449573299708493825 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 17:00:38 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1211112113.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:38 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/748573b9-8d12-4f9c-85fd-84bb9043adbf {"id":"748573b9-8d12-4f9c-85fd-84bb9043adbf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986036} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21eea3cc 748573b9-8d12-4f9c-85fd-84bb9043adbf /tidb/cdc/default/default/upstream/7365808880518281294 {"id":7365808880518281294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/748573b9-8d12-4f9c-85fd-84bb9043adbf {"id":"748573b9-8d12-4f9c-85fd-84bb9043adbf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986036} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21eea3cc 748573b9-8d12-4f9c-85fd-84bb9043adbf /tidb/cdc/default/default/upstream/7365808880518281294 {"id":7365808880518281294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/748573b9-8d12-4f9c-85fd-84bb9043adbf {"id":"748573b9-8d12-4f9c-85fd-84bb9043adbf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986036} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21eea3cc 748573b9-8d12-4f9c-85fd-84bb9043adbf /tidb/cdc/default/default/upstream/7365808880518281294 {"id":7365808880518281294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11314.out cli changefeed create --start-ts=449573298967937025 '--sink-uri=kafka://127.0.0.1:9092/ticdc-partition-table-test-19497?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 103cc861-d21d-497b-bcfb-573c0f9d07f3 Info: {"upstream_id":7365808880518281294,"namespace":"default","id":"103cc861-d21d-497b-bcfb-573c0f9d07f3","sink_uri":"kafka://127.0.0.1:9092/ticdc-partition-table-test-19497?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T17:00:39.344865143+08:00","start_ts":449573298967937025,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573298967937025,"checkpoint_ts":449573298967937025,"checkpoint_time":"2024-05-06 17:00:34.271"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_2 not exists for 5-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.22676.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully + set +x [Mon May 6 17:00:40 CST 2024] <<<<<< START kafka consumer in partition_table case >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: kafka... <<================= table mark.finish_mark_2 not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:41 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f86cf87-050d-482c-92c0-943b5fba76b1 {"id":"1f86cf87-050d-482c-92c0-943b5fba76b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21f9d6cd 1f86cf87-050d-482c-92c0-943b5fba76b1 /tidb/cdc/default/default/upstream/7365808891549305448 {"id":7365808891549305448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f86cf87-050d-482c-92c0-943b5fba76b1 {"id":"1f86cf87-050d-482c-92c0-943b5fba76b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21f9d6cd 1f86cf87-050d-482c-92c0-943b5fba76b1 /tidb/cdc/default/default/upstream/7365808891549305448 {"id":7365808891549305448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1f86cf87-050d-482c-92c0-943b5fba76b1 {"id":"1f86cf87-050d-482c-92c0-943b5fba76b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21f9d6cd 1f86cf87-050d-482c-92c0-943b5fba76b1 /tidb/cdc/default/default/upstream/7365808891549305448 {"id":7365808891549305448,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.cli.33558.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/output_ticdc?protocol=debezium&kafka-version=2.4.0' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:41 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c27a1ff4-152a-42de-a29f-c21cecd734cb {"id":"c27a1ff4-152a-42de-a29f-c21cecd734cb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21ef20d5 c27a1ff4-152a-42de-a29f-c21cecd734cb /tidb/cdc/default/default/upstream/7365808882032325484 {"id":7365808882032325484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c27a1ff4-152a-42de-a29f-c21cecd734cb {"id":"c27a1ff4-152a-42de-a29f-c21cecd734cb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21ef20d5 c27a1ff4-152a-42de-a29f-c21cecd734cb /tidb/cdc/default/default/upstream/7365808882032325484 {"id":7365808882032325484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c27a1ff4-152a-42de-a29f-c21cecd734cb {"id":"c27a1ff4-152a-42de-a29f-c21cecd734cb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21ef20d5 c27a1ff4-152a-42de-a29f-c21cecd734cb /tidb/cdc/default/default/upstream/7365808882032325484 {"id":7365808882032325484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449573299708493825 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.12152.out cli changefeed create --start-ts=449573299708493825 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365808882032325484,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T17:00:42.347773177+08:00","start_ts":449573299708493825,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573299708493825,"checkpoint_ts":449573299708493825,"checkpoint_time":"2024-05-06 17:00:37.096"} PASS Create changefeed successfully! ID: fded779f-8f41-4777-b048-e4d47786d981 Info: {"upstream_id":7365808891549305448,"namespace":"default","id":"fded779f-8f41-4777-b048-e4d47786d981","sink_uri":"kafka://127.0.0.1:9092/output_ticdc?protocol=debezium\u0026kafka-version=2.4.0","create_time":"2024-05-06T17:00:42.442609242+08:00","start_ts":449573301069021188,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"debezium","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-22-ge9f061d7a","resolved_ts":449573301069021188,"checkpoint_ts":449573301069021188,"checkpoint_time":"2024-05-06 17:00:42.286"} PASS + set +x + tso='449573300722991105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449573300722991105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 17:00:42 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2271722719.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.5% of statements in github.com/pingcap/tiflow/... coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_2 not exists for 7-th check, retry later + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3062 0 --:--:-- --:--:-- --:--:-- 3069 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 17:00:37.096","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 17:00:43.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '17:00:37.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '17:00:43.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '17:00:37.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '17:00:43.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + set +x go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4 go: downloading go.uber.org/zap v1.26.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873 go: downloading github.com/segmentio/kafka-go v0.4.45 go: downloading github.com/thessem/zap-prettyconsole v0.3.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/google/go-cmp v0.6.0 go: downloading github.com/google/uuid v1.3.1 go: downloading github.com/alecthomas/chroma v0.10.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/Code-Hex/dd v1.1.0 go: downloading github.com/klauspost/compress v1.17.1 go: downloading github.com/pierrec/lz4/v4 v4.1.15 go: downloading github.com/dlclark/regexp2 v1.4.0 + sink_checkpoint_ts='2024-05-06 17:00:37.096' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '17:00:37.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '17:00:43.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '17:00:37.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '17:00:43.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 17:00:44' + echo 'sink_checkpoint_ts is 2024-05-06' 17:00:37.096 sink_checkpoint_ts is 2024-05-06 17:00:37.096 ++ date -d '2024-05-06 17:00:37.096' +%s + checkpoint_timestamp=1714986037 ++ date -d '2024-05-06 17:00:44' +%s + current_timestamp=1714986044 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 1-th time, retry later The 1 times to try to start tidb cluster... table mark.finish_mark_2 not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 09:00:45 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36645887-253a-44ae-b32e-592651a72fac {"id":"36645887-253a-44ae-b32e-592651a72fac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21faebf2 36645887-253a-44ae-b32e-592651a72fac /tidb/cdc/default/default/upstream/7365808894905280574 {"id":7365808894905280574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36645887-253a-44ae-b32e-592651a72fac {"id":"36645887-253a-44ae-b32e-592651a72fac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21faebf2 36645887-253a-44ae-b32e-592651a72fac /tidb/cdc/default/default/upstream/7365808894905280574 {"id":7365808894905280574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36645887-253a-44ae-b32e-592651a72fac {"id":"36645887-253a-44ae-b32e-592651a72fac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-22-ge9f061d7a","git-hash":"e9f061d7a0898b066e3c6e271294672cae36f6d6","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714986043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d21faebf2 36645887-253a-44ae-b32e-592651a72fac /tidb/cdc/default/default/upstream/7365808894905280574 {"id":7365808894905280574,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x go: downloading github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 go: downloading github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 go: downloading golang.org/x/exp v0.0.0-20231006140011-7918f672742d go: downloading github.com/golang/protobuf v1.5.3 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/prometheus/client_golang v1.17.0 go: downloading github.com/prometheus/client_model v0.5.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e go: downloading github.com/shirou/gopsutil/v3 v3.23.10 go: downloading google.golang.org/protobuf v1.31.0 go: downloading github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071 go: downloading google.golang.org/grpc v1.59.0 go: downloading github.com/prometheus/common v0.45.0 go: downloading github.com/prometheus/procfs v0.12.0 go: downloading github.com/cespare/xxhash/v2 v2.2.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/rogpeppe/go-internal v1.11.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.10 go: downloading go.etcd.io/etcd/client/v3 v3.5.10 go: downloading github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865 go: downloading golang.org/x/sync v0.4.0 go: downloading github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.10 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b go: downloading google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b go: downloading golang.org/x/net v0.18.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b table test.t1 exists + sleep 5 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later [Mon May 6 17:00:47 CST 2024] <<<<<< START kafka consumer in capture_session_done_during_task case >>>>>> lease 22318f4d21faebf2 revoked =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table capture_session_done_during_task.t exists check diff failed 1-th time, retry later table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... table mark.finish_mark_3 not exists for 2-th check, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3800 0 --:--:-- --:--:-- --:--:-- 3796 100 243 100 243 0 0 3793 0 --:--:-- --:--:-- --:--:-- 3796 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 17:00:50.346","puller_resolved_ts":"2024-05-06 17:00:44.145","last_synced_ts":"2024-05-06 17:00:44.196","now_ts":"2024-05-06 17:00:51.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '17:00:50.346","puller_resolved_ts":"2024-05-06' '17:00:44.145","last_synced_ts":"2024-05-06' '17:00:44.196","now_ts":"2024-05-06' '17:00:51.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '17:00:50.346","puller_resolved_ts":"2024-05-06' '17:00:44.145","last_synced_ts":"2024-05-06' '17:00:44.196","now_ts":"2024-05-06' '17:00:51.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table mark.finish_mark_3 not exists for 3-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Aborted by Jenkins Admin Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes table mark.finish_mark_3 not exists for 4-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release {"level":"warn","ts":1714986054.2302194,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00248c000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes Starting Upstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-82-g659f32a813 Edition: Community Git Commit Hash: 659f32a81300f9dbcea9032b3c8e4825555ccfd1 Git Branch: master UTC Build Time: 2024-05-06 07:58:59 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release script returned exit code 143 script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3489ce1c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:11582, start at 2024-05-06 17:00:54.569997961 +0800 CST m=+5.543396467 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:54.578 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:54.584 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:54.584 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later {"level":"warn","ts":1714986055.0434961,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020fe380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes ++ stop_tidb_cluster script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes Starting Upstream TiDB... script returned exit code 143 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3489ce1c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:11582, start at 2024-05-06 17:00:54.569997961 +0800 CST m=+5.543396467 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:54.578 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:54.584 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:54.584 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3489cfac0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1870-4jxkt-1dq73, pid:11667, start at 2024-05-06 17:00:54.645894222 +0800 CST m=+5.564472418 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-17:02:54.653 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-17:00:54.635 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-16:50:54.635 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node script returned exit code 143 {"level":"warn","ts":1714986058.501129,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022bc380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later {"level":"warn","ts":1714986057.2922082,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020561c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate {"level":"warn","ts":1714986053.9596896,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002418540/127.0.0.1:2779","attempt":0,"error":"rpc error: code = Unavailable desc = etcdserver: leader changed"} script returned exit code 143 [Pipeline] // podTemplate [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache script returned exit code 143 [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // cache [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // dir [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G15' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G16' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G17' [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // dir [Pipeline] } {"level":"warn","ts":1714986057.0211444,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002ec2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714986059.0211833,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002ec2000/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} script returned exit code 143 Sending interrupt signal to process Killing processes kill finished with exit code 0 Killing processes kill finished with exit code 0 [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // withCredentials [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // timeout [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] // cache [Pipeline] } [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh: line 1: 10677 Terminated sleep 2m /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh: line 1: 10798 Terminated sleep 2m [Pipeline] // container script returned exit code 143 [Pipeline] // container [Pipeline] // container [Pipeline] // stage [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] } Click here to forcibly terminate running steps [Pipeline] // stage [Pipeline] // dir [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // container [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // container [Pipeline] // cache [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // withEnv [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] // timeout [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // node [Pipeline] // stage [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // podTemplate [Pipeline] // timeout [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // withEnv [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G06' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G12' [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G01' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G00' [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] // stage [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G03' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] // stage [Pipeline] // node [Pipeline] // container [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G07' [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // podTemplate [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: 58178960-5535-4cb0-aaa0-40aaa2dc0e5b Failed in branch Matrix - TEST_GROUP = 'G14' org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: ee31b3b7-4a9e-4e24-b4b4-c36fffcdf3b8 Finished: ABORTED