Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g4vh4-m9089 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1284e6fa2269d8953fce658758ceef86958d1371" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g4vh4" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git rev-list --no-walk cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=45a3db67-c0c3-401b-ae0d-0c9a476e1a76 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g4vh4 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a1283f80f8ebe563bfe9f2bd09649a5cc54c1f9b857850fc6066be7891f3e273 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g4vh4 ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.0' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build1153423411=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 bash' debug command: kubectl -n jenkins-tiflow exec -ti ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-bp209 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 76 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: cf8e99eb5c65c5ea (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 17 IN A 20.205.243.166 ;; Query time: 1 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri Apr 26 10:56:27 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=3&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=3&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=4&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=4&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=5&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=5&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [Makefile, cdc/api/v2/api_helpers.go, cdc/api/v2/api_helpers_mock.go, cdc/api/v2/changefeed.go, cdc/api/v2/changefeed_test.go, cdc/api/v2/model.go, cdc/api/v2/model_test.go, cdc/api/v2/unsafe.go, cdc/entry/codec.go, cdc/entry/codec_test.go, cdc/entry/mounter.go, cdc/entry/mounter_test.go, cdc/entry/schema/snapshot.go, cdc/entry/schema/snapshot_test.go, cdc/entry/schema_storage.go, cdc/entry/schema_storage_test.go, cdc/entry/schema_test_helper.go, cdc/entry/validator.go, cdc/kv/client_bench_test.go, cdc/kv/client_test.go, cdc/kv/shared_client_test.go, cdc/kv/store_op.go, cdc/model/changefeed_test.go, cdc/model/codec/codec.go, cdc/model/codec/codec_test.go, cdc/model/codec/v1/codec.go, cdc/model/codec/v1/convert.go, cdc/model/codec/v1/convert_test.go, cdc/model/mounter.go, cdc/model/owner.go, cdc/model/schema_storage.go, cdc/model/schema_storage_test.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/owner/changefeed.go, cdc/owner/changefeed_test.go, cdc/owner/ddl_manager.go, cdc/owner/ddl_manager_test.go, cdc/owner/ddl_sink.go, cdc/owner/schema.go, cdc/owner/schema_test.go, cdc/processor/sourcemanager/engine/factory/factory.go, cdc/processor/sourcemanager/engine/factory/pebble.go, cdc/processor/sourcemanager/engine/pebble/db.go, cdc/processor/sourcemanager/engine/pebble/db_test.go, cdc/processor/sourcemanager/engine/pebble/event_sorter.go, cdc/processor/sourcemanager/engine/pebble/event_sorter_test.go, cdc/puller/ddl_puller.go, cdc/puller/ddl_puller_test.go, cdc/puller/puller.go, cdc/puller/puller_test.go, cdc/server/server.go, cdc/sink/ddlsink/cloudstorage/cloud_storage_ddl_sink.go, cdc/sink/ddlsink/cloudstorage/cloud_storage_ddl_sink_test.go, cdc/sink/ddlsink/mq/mq_ddl_sink_test.go, cdc/sink/ddlsink/mq/pulsar_ddl_sink_test.go, cdc/sink/ddlsink/mysql/mysql_ddl_sink.go, cdc/sink/ddlsink/mysql/mysql_ddl_sink_test.go, cdc/sink/dmlsink/cloudstorage/cloud_storage_dml_sink_test.go, cdc/sink/dmlsink/cloudstorage/defragmenter_test.go, cdc/sink/dmlsink/cloudstorage/dml_worker_test.go, cdc/sink/dmlsink/cloudstorage/encoding_worker_test.go, cdc/sink/dmlsink/mq/dispatcher/event_router.go, cdc/sink/dmlsink/mq/dispatcher/event_router_test.go, cdc/sink/dmlsink/mq/dispatcher/partition/columns_test.go, cdc/sink/dmlsink/mq/dispatcher/partition/index_value_test.go, cdc/sink/dmlsink/mq/transformer/columnselector/column_selector.go, cdc/sink/dmlsink/txn/event.go, cdc/sink/dmlsink/txn/event_test.go, cdc/sink/dmlsink/txn/mysql/dml.go, cdc/sink/dmlsink/txn/mysql/dml_test.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/cdc/main.go, cmd/dm-syncer/config.go, cmd/dm-worker/main.go, cmd/filter-helper/main.go, cmd/kafka-consumer/main.go, dm/chaos/cases/db.go, dm/chaos/cases/diff.go, dm/chaos/cases/schema.go, dm/chaos/cases/stmt.go, dm/chaos/cases/task.go, dm/checker/check_test.go, dm/checker/checker.go, dm/config/subtask.go, dm/config/subtask_test.go, dm/config/task.go, dm/config/task_converters.go, dm/config/task_converters_test.go, dm/config/task_test.go, dm/ctl/common/util.go, dm/dumpling/dumpling.go, dm/dumpling/dumpling_test.go, dm/loader/checkpoint.go, dm/loader/lightning.go, dm/loader/lightning_test.go, dm/master/config.go, dm/master/config_test.go, dm/master/openapi_view.go, dm/master/server.go, dm/master/server_test.go, dm/master/shardddl/optimist.go, dm/master/shardddl/optimist_test.go, dm/pkg/binlog/event/util.go, dm/pkg/checker/binlog.go, dm/pkg/checker/binlog_test.go, dm/pkg/checker/conn_checker.go, dm/pkg/checker/lightning.go, dm/pkg/checker/lightning_test.go, dm/pkg/checker/mysql_server.go, dm/pkg/checker/mysql_server_test.go, dm/pkg/checker/onlineddl.go, dm/pkg/checker/privilege.go, dm/pkg/checker/privilege_test.go, dm/pkg/checker/table_structure.go, dm/pkg/checker/table_structure_test.go, dm/pkg/checker/utils.go, dm/pkg/checker/utils_test.go, dm/pkg/conn/baseconn.go, dm/pkg/conn/baseconn_test.go, dm/pkg/conn/basedb.go, dm/pkg/conn/db.go, dm/pkg/conn/db_test.go, dm/pkg/conn/mockdb.go, dm/pkg/conn/utils.go, dm/pkg/election/election_test.go, dm/pkg/etcdutil/etcdutil_test.go, dm/pkg/log/log.go, dm/pkg/log/log_test.go, dm/pkg/parser/common.go, dm/pkg/parser/common_test.go, dm/pkg/retry/errors.go, dm/pkg/retry/strategy_test.go, dm/pkg/schema/tracker.go, dm/pkg/schema/tracker_test.go, dm/pkg/schema/visitor.go, dm/pkg/shardddl/optimism/info.go, dm/pkg/shardddl/optimism/info_test.go, dm/pkg/shardddl/optimism/keeper.go, dm/pkg/shardddl/optimism/keeper_test.go, dm/pkg/shardddl/optimism/lock.go, dm/pkg/shardddl/optimism/lock_test.go, dm/pkg/shardddl/pessimism/info_test.go, dm/pkg/upgrade/upgrade.go, dm/pkg/utils/common.go, dm/pkg/utils/common_test.go, dm/pkg/utils/time.go, dm/pkg/utils/util.go, dm/pkg/utils/util_test.go, dm/pkg/v1dbschema/schema.go, dm/relay/file_util.go, dm/relay/file_util_test.go, dm/relay/relay.go, dm/relay/relay_test.go, dm/simulator/config/config.go, dm/simulator/sqlgen/impl.go, dm/simulator/sqlgen/impl_test.go, dm/syncer/causality.go, dm/syncer/checkpoint.go, dm/syncer/checkpoint_flush_worker.go, dm/syncer/checkpoint_test.go, dm/syncer/compactor_test.go, dm/syncer/data_validator.go, dm/syncer/data_validator_test.go, dm/syncer/dbconn/db.go, dm/syncer/dbconn/upstream_db.go, dm/syncer/dbconn/utils.go, dm/syncer/ddl.go, dm/syncer/ddl_test.go, dm/syncer/dml.go, dm/syncer/dml_test.go, dm/syncer/dml_worker_test.go, dm/syncer/error.go, dm/syncer/error_test.go, dm/syncer/expr_filter_group.go, dm/syncer/expr_filter_group_test.go, dm/syncer/filter.go, dm/syncer/filter_test.go, dm/syncer/handle_error.go, dm/syncer/job.go, dm/syncer/job_test.go, dm/syncer/online-ddl-tools/online_ddl.go, dm/syncer/opt_sharding_group.go, dm/syncer/optimist.go, dm/syncer/safe-mode/mode.go, dm/syncer/safe-mode/mode_test.go, dm/syncer/schema.go, dm/syncer/shardddl/optimist.go, dm/syncer/shardddl/optimist_test.go, dm/syncer/sharding-meta/shardmeta.go, dm/syncer/sharding_group.go, dm/syncer/sharding_group_test.go, dm/syncer/status_test.go, dm/syncer/syncer.go, dm/syncer/syncer_test.go, dm/syncer/util.go, dm/syncer/util_test.go, dm/syncer/validate_worker.go, dm/syncer/validate_worker_test.go, dm/syncer/validator_checkpoint.go, dm/syncer/validator_checkpoint_test.go, dm/syncer/validator_cond.go, dm/syncer/validator_cond_test.go, dm/tests/incompatible_ddl_changes/conf/dm-task1.yaml, dm/tests/incompatible_ddl_changes/run.sh, dm/tests/lightning_mode/run.sh, dm/tests/tls/run.sh, dm/unit/unit_test.go, dm/worker/server_test.go, dm/worker/task_checker_test.go, engine/chaos/cases/dm/case.go, engine/chaos/cases/dm/db.go, engine/executor/server.go, engine/jobmaster/dm/checkpoint/agent.go, engine/jobmaster/dm/checkpoint/agent_test.go, engine/jobmaster/dm/config/config.go, engine/jobmaster/dm/ddl_coordinator.go, engine/pkg/meta/internal/etcdkv/mock.go, engine/pkg/meta/model/config.go, engine/pkg/orm/util_test.go, examples/golang/avro-checksum-verification/go.mod, examples/golang/avro-checksum-verification/go.sum, examples/golang/avro-checksum-verification/main.go, go.mod, go.sum, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/cmd/server/server.go, pkg/cmd/server/server_test.go, pkg/config/filter.go, pkg/config/outdated/v1.go, pkg/config/replica_config.go, pkg/errorutil/util.go, pkg/errorutil/util_test.go, pkg/etcd/etcd.go, pkg/filter/expr_filter.go, pkg/filter/expr_filter_test.go, pkg/filter/filter.go, pkg/filter/filter_test.go, pkg/filter/filter_test_helper.go, pkg/filter/sql_event_filter.go, pkg/filter/utils.go, pkg/filter/utils_test.go, pkg/httputil/httputil_test.go, pkg/migrate/migrate_test.go, pkg/pdutil/api_client.go, pkg/pdutil/api_client_test.go, pkg/pdutil/utils_test.go, pkg/sink/cloudstorage/path_test.go, pkg/sink/cloudstorage/table_definition.go, pkg/sink/cloudstorage/table_definition_test.go, pkg/sink/codec/avro/avro.go, pkg/sink/codec/avro/avro_test.go, pkg/sink/codec/avro/decoder.go, pkg/sink/codec/avro/decoder_test.go, pkg/sink/codec/canal/canal_encoder_test.go, pkg/sink/codec/canal/canal_entry.go, pkg/sink/codec/canal/canal_entry_test.go, pkg/sink/codec/canal/canal_json_message.go, pkg/sink/codec/canal/canal_json_row_event_encoder_test.go, pkg/sink/codec/canal/canal_json_txn_event_encoder_test.go, pkg/sink/codec/canal/canal_test_util.go, pkg/sink/codec/common/message_test.go, pkg/sink/codec/common/verify_checksum.go, pkg/sink/codec/craft/craft_encoder_test.go, pkg/sink/codec/craft/message_decoder.go, pkg/sink/codec/craft/message_encoder.go, pkg/sink/codec/csv/csv_decoder.go, pkg/sink/codec/csv/csv_decoder_test.go, pkg/sink/codec/csv/csv_encoder_test.go, pkg/sink/codec/csv/csv_message.go, pkg/sink/codec/csv/csv_message_test.go, pkg/sink/codec/internal/batch_tester.go, pkg/sink/codec/internal/column.go, pkg/sink/codec/internal/java.go, pkg/sink/codec/maxwell/maxwell_encoder_test.go, pkg/sink/codec/maxwell/maxwell_message.go, pkg/sink/codec/maxwell/maxwell_message_test.go, pkg/sink/codec/open/open_protocol_decoder.go, pkg/sink/codec/open/open_protocol_encoder_test.go, pkg/sink/codec/open/open_protocol_message.go, pkg/sink/codec/open/open_protocol_message_test.go, pkg/sink/kafka/claimcheck/claim_check_test.go, pkg/sink/mysql/db_helper.go, pkg/spanz/span.go, pkg/spanz/span_test.go, pkg/sqlmodel/causality.go, pkg/sqlmodel/row_change.go, pkg/sqlmodel/row_change_test.go, pkg/sqlmodel/utils.go, pkg/sqlmodel/where_handle.go, pkg/sqlmodel/where_handle_test.go, pkg/txnutil/gc/gc_manager.go, pkg/txnutil/gc/gc_manager_test.go, pkg/upstream/upstream.go, pkg/util/memory.go, pkg/util/tz.go, pkg/version/check.go, pkg/version/check_test.go, tests/integration_tests/cdc/dailytest/db.go, tests/integration_tests/cdc/dailytest/job.go, tests/integration_tests/cdc/dailytest/parser.go, tests/integration_tests/resolve_lock/main.go, tests/integration_tests/util/db.go] [Pipeline] echo diff file not matched: Makefile [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-823a389-aa323aa) 199979520 bytes in 0.93 secs (215530585 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/.git/ .git HEAD is now at 8b79396bb Merge commit 'aa323aa99660421afc797ea115dbb30242e573cf' into HEAD POST git-upload-pack (686 bytes) From https://github.com/pingcap/tiflow = [up to date] release-7.5 -> origin/release-7.5 = [up to date] refs/pull/10970/head -> origin/pr/10970/head Warning: you are leaving 1 commit behind, not connected to any of your branches: 8b79396bb Merge commit 'aa323aa99660421afc797ea115dbb30242e573cf' into HEAD If you want to keep it by creating a new branch, this may be a good time to do so with: git branch 8b79396bb HEAD is now at 823a3899c sink(ticdc): Revert changes related to the conflict detector (#10924) ๐Ÿšง Checkouting to base SHA:823a3899cc17bf14ad7875ab0cc69092186abc4a... HEAD is now at 823a3899c sink(ticdc): Revert changes related to the conflict detector (#10924) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 823a3899cc17bf14ad7875ab0cc69092186abc4a 823a3899c sink(ticdc): Revert changes related to the conflict detector (#10924) 3c4a12d87 mysql(ticdc): fix the data race in the conflict detector (#10630) (#10642) 0d6a6613d kvclient(ticdc): close the grpc client after all goroutine exited to prevent data race and panic (#10865) (#10906) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 823a3899cc17bf14ad7875ab0cc69092186abc4a ... Merge made by the 'ort' strategy. Makefile | 2 +- cdc/api/v2/api_helpers.go | 2 +- cdc/api/v2/api_helpers_mock.go | 2 +- cdc/api/v2/changefeed.go | 2 +- cdc/api/v2/changefeed_test.go | 2 +- cdc/api/v2/model.go | 2 +- cdc/api/v2/model_test.go | 2 +- cdc/api/v2/unsafe.go | 2 +- cdc/entry/codec.go | 12 +- cdc/entry/codec_test.go | 6 +- cdc/entry/mounter.go | 38 +- cdc/entry/mounter_test.go | 60 +- cdc/entry/schema/snapshot.go | 4 +- cdc/entry/schema/snapshot_test.go | 2 +- cdc/entry/schema_storage.go | 4 +- cdc/entry/schema_storage_test.go | 24 +- cdc/entry/schema_test_helper.go | 18 +- cdc/entry/validator.go | 2 +- cdc/kv/client_bench_test.go | 2 +- cdc/kv/client_test.go | 4 +- cdc/kv/shared_client_test.go | 2 +- cdc/kv/store_op.go | 6 +- cdc/model/changefeed_test.go | 2 +- cdc/model/codec/codec.go | 2 +- cdc/model/codec/codec_test.go | 6 +- cdc/model/codec/v1/codec.go | 4 +- cdc/model/codec/v1/convert.go | 2 +- cdc/model/codec/v1/convert_test.go | 6 +- cdc/model/mounter.go | 2 +- cdc/model/owner.go | 2 +- cdc/model/schema_storage.go | 10 +- cdc/model/schema_storage_test.go | 6 +- cdc/model/sink.go | 6 +- cdc/model/sink_test.go | 6 +- cdc/owner/changefeed.go | 2 +- cdc/owner/changefeed_test.go | 2 +- cdc/owner/ddl_manager.go | 2 +- cdc/owner/ddl_manager_test.go | 2 +- cdc/owner/ddl_sink.go | 6 +- cdc/owner/schema.go | 6 +- cdc/owner/schema_test.go | 21 +- .../sourcemanager/engine/factory/factory.go | 6 +- .../sourcemanager/engine/factory/pebble.go | 39 +- cdc/processor/sourcemanager/engine/pebble/db.go | 13 +- .../sourcemanager/engine/pebble/db_test.go | 1 + .../sourcemanager/engine/pebble/event_sorter.go | 6 +- .../engine/pebble/event_sorter_test.go | 8 +- cdc/puller/ddl_puller.go | 6 +- cdc/puller/ddl_puller_test.go | 6 +- cdc/puller/puller.go | 2 +- cdc/puller/puller_test.go | 4 +- cdc/server/server.go | 2 +- .../ddlsink/cloudstorage/cloud_storage_ddl_sink.go | 2 +- .../cloudstorage/cloud_storage_ddl_sink_test.go | 6 +- cdc/sink/ddlsink/mq/mq_ddl_sink_test.go | 2 +- cdc/sink/ddlsink/mq/pulsar_ddl_sink_test.go | 2 +- cdc/sink/ddlsink/mysql/mysql_ddl_sink.go | 2 +- cdc/sink/ddlsink/mysql/mysql_ddl_sink_test.go | 4 +- .../cloudstorage/cloud_storage_dml_sink_test.go | 8 +- cdc/sink/dmlsink/cloudstorage/defragmenter_test.go | 6 +- cdc/sink/dmlsink/cloudstorage/dml_worker_test.go | 6 +- .../dmlsink/cloudstorage/encoding_worker_test.go | 6 +- cdc/sink/dmlsink/mq/dispatcher/event_router.go | 2 +- .../dmlsink/mq/dispatcher/event_router_test.go | 2 +- .../mq/dispatcher/partition/columns_test.go | 2 +- .../mq/dispatcher/partition/index_value_test.go | 2 +- .../transformer/columnselector/column_selector.go | 2 +- cdc/sink/dmlsink/txn/event.go | 2 +- cdc/sink/dmlsink/txn/event_test.go | 2 +- cdc/sink/dmlsink/txn/mysql/dml.go | 2 +- cdc/sink/dmlsink/txn/mysql/dml_test.go | 4 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 12 +- cmd/cdc/main.go | 2 +- cmd/dm-syncer/config.go | 4 +- cmd/dm-worker/main.go | 3 + cmd/filter-helper/main.go | 2 +- cmd/kafka-consumer/main.go | 2 +- dm/chaos/cases/db.go | 4 +- dm/chaos/cases/diff.go | 2 +- dm/chaos/cases/schema.go | 4 +- dm/chaos/cases/stmt.go | 4 +- dm/chaos/cases/task.go | 4 +- dm/checker/check_test.go | 4 +- dm/checker/checker.go | 50 +- dm/config/subtask.go | 8 +- dm/config/subtask_test.go | 2 +- dm/config/task.go | 8 +- dm/config/task_converters.go | 4 +- dm/config/task_converters_test.go | 2 +- dm/config/task_test.go | 4 +- dm/ctl/common/util.go | 2 +- dm/dumpling/dumpling.go | 6 +- dm/dumpling/dumpling_test.go | 6 +- dm/loader/checkpoint.go | 2 +- dm/loader/lightning.go | 46 +- dm/loader/lightning_test.go | 4 +- dm/master/config.go | 10 +- dm/master/config_test.go | 16 +- dm/master/openapi_view.go | 2 +- dm/master/server.go | 2 +- dm/master/server_test.go | 14 +- dm/master/shardddl/optimist.go | 2 +- dm/master/shardddl/optimist_test.go | 14 +- dm/pkg/binlog/event/util.go | 6 +- dm/pkg/checker/binlog.go | 5 +- dm/pkg/checker/binlog_test.go | 2 +- dm/pkg/checker/conn_checker.go | 4 +- dm/pkg/checker/lightning.go | 20 +- dm/pkg/checker/lightning_test.go | 2 +- dm/pkg/checker/mysql_server.go | 2 +- dm/pkg/checker/mysql_server_test.go | 2 +- dm/pkg/checker/onlineddl.go | 4 +- dm/pkg/checker/privilege.go | 14 +- dm/pkg/checker/privilege_test.go | 4 +- dm/pkg/checker/table_structure.go | 15 +- dm/pkg/checker/table_structure_test.go | 2 +- dm/pkg/checker/utils.go | 6 +- dm/pkg/checker/utils_test.go | 4 +- dm/pkg/conn/baseconn.go | 2 +- dm/pkg/conn/baseconn_test.go | 2 +- dm/pkg/conn/basedb.go | 4 +- dm/pkg/conn/db.go | 10 +- dm/pkg/conn/db_test.go | 8 +- dm/pkg/conn/mockdb.go | 16 +- dm/pkg/conn/utils.go | 2 +- dm/pkg/election/election_test.go | 8 +- dm/pkg/etcdutil/etcdutil_test.go | 20 +- dm/pkg/log/log.go | 4 +- dm/pkg/log/log_test.go | 4 +- dm/pkg/parser/common.go | 12 +- dm/pkg/parser/common_test.go | 4 +- dm/pkg/retry/errors.go | 2 +- dm/pkg/retry/strategy_test.go | 2 +- dm/pkg/schema/tracker.go | 43 +- dm/pkg/schema/tracker_test.go | 14 +- dm/pkg/schema/visitor.go | 4 +- dm/pkg/shardddl/optimism/info.go | 4 +- dm/pkg/shardddl/optimism/info_test.go | 12 +- dm/pkg/shardddl/optimism/keeper.go | 2 +- dm/pkg/shardddl/optimism/keeper_test.go | 6 +- dm/pkg/shardddl/optimism/lock.go | 10 +- dm/pkg/shardddl/optimism/lock_test.go | 10 +- dm/pkg/shardddl/pessimism/info_test.go | 2 +- dm/pkg/upgrade/upgrade.go | 2 +- dm/pkg/utils/common.go | 41 +- dm/pkg/utils/common_test.go | 2 +- dm/pkg/utils/time.go | 6 +- dm/pkg/utils/util.go | 2 +- dm/pkg/utils/util_test.go | 2 +- dm/pkg/v1dbschema/schema.go | 4 +- dm/relay/file_util.go | 2 +- dm/relay/file_util_test.go | 2 +- dm/relay/relay.go | 5 +- dm/relay/relay_test.go | 2 +- dm/simulator/config/config.go | 2 +- dm/simulator/sqlgen/impl.go | 10 +- dm/simulator/sqlgen/impl_test.go | 4 +- dm/syncer/causality.go | 2 +- dm/syncer/checkpoint.go | 8 +- dm/syncer/checkpoint_flush_worker.go | 2 +- dm/syncer/checkpoint_test.go | 8 +- dm/syncer/compactor_test.go | 4 +- dm/syncer/data_validator.go | 4 +- dm/syncer/data_validator_test.go | 6 +- dm/syncer/dbconn/db.go | 2 +- dm/syncer/dbconn/upstream_db.go | 6 +- dm/syncer/dbconn/utils.go | 4 +- dm/syncer/ddl.go | 58 +- dm/syncer/ddl_test.go | 10 +- dm/syncer/dml.go | 12 +- dm/syncer/dml_test.go | 16 +- dm/syncer/dml_worker_test.go | 10 +- dm/syncer/error.go | 16 +- dm/syncer/error_test.go | 2 +- dm/syncer/expr_filter_group.go | 20 +- dm/syncer/expr_filter_group_test.go | 6 +- dm/syncer/filter.go | 2 +- dm/syncer/filter_test.go | 4 +- dm/syncer/handle_error.go | 4 +- dm/syncer/job.go | 2 +- dm/syncer/job_test.go | 2 +- dm/syncer/online-ddl-tools/online_ddl.go | 10 +- dm/syncer/opt_sharding_group.go | 2 +- dm/syncer/optimist.go | 2 +- dm/syncer/safe-mode/mode.go | 2 +- dm/syncer/safe-mode/mode_test.go | 2 +- dm/syncer/schema.go | 14 +- dm/syncer/shardddl/optimist.go | 4 +- dm/syncer/shardddl/optimist_test.go | 12 +- dm/syncer/sharding-meta/shardmeta.go | 4 +- dm/syncer/sharding_group.go | 4 +- dm/syncer/sharding_group_test.go | 4 +- dm/syncer/status_test.go | 4 +- dm/syncer/syncer.go | 18 +- dm/syncer/syncer_test.go | 14 +- dm/syncer/util.go | 7 +- dm/syncer/util_test.go | 10 +- dm/syncer/validate_worker.go | 8 +- dm/syncer/validate_worker_test.go | 10 +- dm/syncer/validator_checkpoint.go | 4 +- dm/syncer/validator_checkpoint_test.go | 6 +- dm/syncer/validator_cond.go | 2 +- dm/syncer/validator_cond_test.go | 10 +- .../incompatible_ddl_changes/conf/dm-task1.yaml | 28 +- dm/tests/incompatible_ddl_changes/run.sh | 96 +- dm/tests/lightning_mode/run.sh | 2 +- dm/tests/tls/run.sh | 9 +- dm/unit/unit_test.go | 4 +- dm/worker/server_test.go | 8 +- dm/worker/task_checker_test.go | 2 +- engine/chaos/cases/dm/case.go | 4 +- engine/chaos/cases/dm/db.go | 2 +- engine/executor/server.go | 4 +- engine/jobmaster/dm/checkpoint/agent.go | 14 +- engine/jobmaster/dm/checkpoint/agent_test.go | 7 +- engine/jobmaster/dm/config/config.go | 4 +- engine/jobmaster/dm/ddl_coordinator.go | 10 +- engine/pkg/meta/internal/etcdkv/mock.go | 4 +- engine/pkg/meta/model/config.go | 6 +- engine/pkg/orm/util_test.go | 2 +- examples/golang/avro-checksum-verification/go.mod | 79 +- examples/golang/avro-checksum-verification/go.sum | 311 +++-- examples/golang/avro-checksum-verification/main.go | 23 +- go.mod | 276 ++--- go.sum | 1227 ++++++-------------- pkg/applier/redo.go | 2 +- pkg/applier/redo_test.go | 2 +- pkg/cmd/server/server.go | 2 +- pkg/cmd/server/server_test.go | 2 +- pkg/config/filter.go | 2 +- pkg/config/outdated/v1.go | 2 +- pkg/config/replica_config.go | 2 +- pkg/errorutil/util.go | 8 +- pkg/errorutil/util_test.go | 4 +- pkg/etcd/etcd.go | 4 +- pkg/filter/expr_filter.go | 24 +- pkg/filter/expr_filter_test.go | 14 +- pkg/filter/filter.go | 4 +- pkg/filter/filter_test.go | 2 +- pkg/filter/filter_test_helper.go | 18 +- pkg/filter/sql_event_filter.go | 6 +- pkg/filter/utils.go | 8 +- pkg/filter/utils_test.go | 18 +- pkg/httputil/httputil_test.go | 2 +- pkg/migrate/migrate_test.go | 4 +- pkg/pdutil/api_client.go | 6 +- pkg/pdutil/api_client_test.go | 6 +- pkg/pdutil/utils_test.go | 6 +- pkg/sink/cloudstorage/path_test.go | 6 +- pkg/sink/cloudstorage/table_definition.go | 8 +- pkg/sink/cloudstorage/table_definition_test.go | 8 +- pkg/sink/codec/avro/avro.go | 8 +- pkg/sink/codec/avro/avro_test.go | 6 +- pkg/sink/codec/avro/decoder.go | 32 +- pkg/sink/codec/avro/decoder_test.go | 15 +- pkg/sink/codec/canal/canal_encoder_test.go | 2 +- pkg/sink/codec/canal/canal_entry.go | 6 +- pkg/sink/codec/canal/canal_entry_test.go | 4 +- pkg/sink/codec/canal/canal_json_message.go | 4 +- .../canal/canal_json_row_event_encoder_test.go | 2 +- .../canal/canal_json_txn_event_encoder_test.go | 2 +- pkg/sink/codec/canal/canal_test_util.go | 4 +- pkg/sink/codec/common/message_test.go | 6 +- pkg/sink/codec/common/verify_checksum.go | 190 +++ pkg/sink/codec/craft/craft_encoder_test.go | 2 +- pkg/sink/codec/craft/message_decoder.go | 4 +- pkg/sink/codec/craft/message_encoder.go | 2 +- pkg/sink/codec/csv/csv_decoder.go | 6 +- pkg/sink/codec/csv/csv_decoder_test.go | 6 +- pkg/sink/codec/csv/csv_encoder_test.go | 6 +- pkg/sink/codec/csv/csv_message.go | 10 +- pkg/sink/codec/csv/csv_message_test.go | 10 +- pkg/sink/codec/internal/batch_tester.go | 2 +- pkg/sink/codec/internal/column.go | 2 +- pkg/sink/codec/internal/java.go | 2 +- pkg/sink/codec/maxwell/maxwell_encoder_test.go | 4 +- pkg/sink/codec/maxwell/maxwell_message.go | 4 +- pkg/sink/codec/maxwell/maxwell_message_test.go | 2 +- pkg/sink/codec/open/open_protocol_decoder.go | 4 +- pkg/sink/codec/open/open_protocol_encoder_test.go | 4 +- pkg/sink/codec/open/open_protocol_message.go | 2 +- pkg/sink/codec/open/open_protocol_message_test.go | 2 +- pkg/sink/kafka/claimcheck/claim_check_test.go | 2 +- pkg/sink/mysql/db_helper.go | 4 +- pkg/spanz/span.go | 8 +- pkg/spanz/span_test.go | 2 +- pkg/sqlmodel/causality.go | 8 +- pkg/sqlmodel/row_change.go | 4 +- pkg/sqlmodel/row_change_test.go | 13 +- pkg/sqlmodel/utils.go | 2 +- pkg/sqlmodel/where_handle.go | 6 +- pkg/sqlmodel/where_handle_test.go | 6 +- pkg/txnutil/gc/gc_manager.go | 8 +- pkg/txnutil/gc/gc_manager_test.go | 2 +- pkg/upstream/upstream.go | 2 +- pkg/util/memory.go | 2 +- pkg/util/tz.go | 31 +- pkg/version/check.go | 2 +- pkg/version/check_test.go | 2 +- tests/integration_tests/cdc/dailytest/db.go | 2 +- tests/integration_tests/cdc/dailytest/job.go | 4 +- tests/integration_tests/cdc/dailytest/parser.go | 8 +- tests/integration_tests/resolve_lock/main.go | 8 +- tests/integration_tests/util/db.go | 2 +- 305 files changed, 2043 insertions(+), 2191 deletions(-) create mode 100644 pkg/sink/codec/common/verify_checksum.go ๐Ÿงพ Pre-merged result: 980f60a5b0a82befa601d05e3558ec8795625b13 980f60a5b Merge commit 'aa323aa99660421afc797ea115dbb30242e573cf' into HEAD aa323aa99 fix potential typos. 13c5e9410 update โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-823a389-aa323aa already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh release-7.5 Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 594 0 --:--:-- --:--:-- --:--:-- 602 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2232 0 --:--:-- --:--:-- --:--:-- 2277 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1433 0 --:--:-- --:--:-- --:--:-- 1464 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2045 0 --:--:-- --:--:-- --:--:-- 2157 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/e406d5780b18a1f2aaf6230cde4b6403991e228d/centos7/tidb-server.tar.gz 2024-04-26 18:56:53 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/e406d5780b18a1f2aaf6230cde4b6403991e228d/centos7/tidb-server.tar.gz [511580034/511580034] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/58453365285465cd90bc4472cff2bad7ce4d764b/centos7/pd-server.tar.gz 2024-04-26 18:57:09 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/58453365285465cd90bc4472cff2bad7ce4d764b/centos7/pd-server.tar.gz [173219013/173219013] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990/centos7/tikv-server.tar.gz 2024-04-26 18:57:39 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990/centos7/tikv-server.tar.gz [810252984/810252984] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/9002cc34d3b593a718b6c5260ba18f30a45ab314/centos7/tiflash.tar.gz 2024-04-26 18:57:56 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/release-7.5/9002cc34d3b593a718b6c5260ba18f30a45ab314/centos7/tiflash.tar.gz [263167205/263167205] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-04-26 18:57:59 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-04-26 18:58:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-04-26 18:58:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-04-26 18:58:03 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-04-26 18:58:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-04-26 18:58:13 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.7G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 26 18:58 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 18:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 18:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 18:58 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 18 15:55 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 18 15:55 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 18 15:55 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 18 15:55 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 18 15:25 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 88M Apr 18 15:48 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M Apr 3 18:05 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 42M Apr 3 18:05 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M Apr 3 18:05 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 3 18:05 pd-recover -rwxr-xr-x. 1 jenkins jenkins 102M Apr 3 18:04 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M Apr 3 18:05 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 3 18:05 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 29M Apr 3 18:05 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M Apr 22 17:56 tidb-server -rwxr-xr-x. 1 jenkins jenkins 377M Apr 18 15:56 tiflash -rwxr-xr-x. 1 jenkins jenkins 353M Apr 25 18:18 tikv-server + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.7G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 26 18:58 . drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 18:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 18:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 18:58 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 18 15:55 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 18 15:55 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 18 15:55 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 18 15:55 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 18 15:25 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 88M Apr 18 15:48 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M Apr 3 18:05 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 42M Apr 3 18:05 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M Apr 3 18:05 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 3 18:05 pd-recover -rwxr-xr-x. 1 jenkins jenkins 102M Apr 3 18:04 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M Apr 3 18:05 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 3 18:05 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 29M Apr 3 18:05 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M Apr 22 17:56 tidb-server -rwxr-xr-x. 1 jenkins jenkins 377M Apr 18 15:56 tiflash -rwxr-xr-x. 1 jenkins jenkins 353M Apr 25 18:18 tikv-server + ./bin/tidb-server -V Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 + ./bin/tikv-server -V TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 18:58 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 18:58 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 10:58:16" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -o bin/cdc ./cmd/cdc/main.go go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/gin-gonic/gin v1.8.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/robfig/cron v1.2.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20230619063055-fd67d94318fd go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading blainsmith.com/go/seahash v1.2.1 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/go-playground/validator/v10 v10.11.1 go: downloading github.com/pelletier/go-toml/v2 v2.0.5 go: downloading github.com/ugorji/go/codec v1.2.7 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/swaggo/swag v1.8.3 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.8 go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/go-playground/universal-translator v0.18.0 go: downloading github.com/leodido/go-urn v1.2.1 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/jsonreference v0.20.2 go: downloading github.com/go-openapi/spec v0.20.6 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/fn v1.0.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-playground/locales v0.14.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/go-openapi/jsonpointer v0.19.6 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/go-openapi/swag v0.22.3 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:02:18" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:02:26" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:02:34" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:02:34" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:02:34" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading gorm.io/gorm v1.23.8 go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/glebarez/sqlite v1.4.6 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/glebarez/go-sqlite v1.17.3 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading modernc.org/sqlite v1.17.3 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.16.8 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading modernc.org/memory v1.1.1 go: downloading modernc.org/mathutil v1.6.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:02:34" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g980f60a5b" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:02:34" -X "github.com/pingcap/tiflow/pkg/version.GitHash=980f60a5b0a82befa601d05e3558ec8795625b13" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g980f60a5b"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.1G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 19:08 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 18:58 .. -rwxr-xr-x. 1 jenkins jenkins 204M Apr 26 19:08 cdc -rwxr-xr-x. 1 jenkins jenkins 335M Apr 26 19:08 cdc.test -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 19:02 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 19:03 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 19:02 cdc_storage_consumer + ./bin/cdc version Release Version: v7.5.1-57-g980f60a5b Git Commit Hash: 980f60a5b0a82befa601d05e3558ec8795625b13 Git Branch: HEAD UTC Build Time: 2024-04-26 11:02:34 Go Version: go version go1.21.0 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-823a389-aa323aa already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.7G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 26 19:08 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 18:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 26 19:08 bin -rwxr-xr-x. 1 jenkins jenkins 204M Apr 26 19:08 cdc -rwxr-xr-x. 1 jenkins jenkins 335M Apr 26 19:08 cdc.test -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 19:02 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 19:03 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M Apr 26 19:02 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:08 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 26 19:08 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 26 19:08 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 26 19:08 jq drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:08 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:08 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Apr 26 19:08 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 26 19:08 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Apr 26 19:08 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 26 19:08 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 26 19:08 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 26 19:08 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 88M Apr 26 19:08 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Apr 26 19:08 minio -rwxr-xr-x. 1 jenkins jenkins 33M Apr 26 19:08 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 42M Apr 26 19:08 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M Apr 26 19:08 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 26 19:08 pd-recover -rwxr-xr-x. 1 jenkins jenkins 102M Apr 26 19:08 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M Apr 26 19:08 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 26 19:08 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:08 share -rwxr-xr-x. 1 jenkins jenkins 29M Apr 26 19:08 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Apr 26 19:08 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M Apr 26 19:08 tidb-server -rwxr-xr-x. 1 jenkins jenkins 377M Apr 26 19:08 tiflash -rwxr-xr-x. 1 jenkins jenkins 353M Apr 26 19:08 tikv-server [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 101.13 secs (33245931 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9 [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-q7ckx-08vkq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "98208152f393c8d5fa91def8061d0acffbad817a" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-q7ckx" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@425c6191; decorates RemoteLauncher[hudson.remoting.Channel@69458703:JNLP4-connect connection from 10.233.126.177/10.233.126.177:54542] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt [Pipeline] node Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g8645-4fnps --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a602ab9dbddcc99024b8059b99cf21a63213fe29" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g8645" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j Avoid second fetch [Pipeline] node Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] cache Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@eaf2c9c; decorates RemoteLauncher[hudson.remoting.Channel@487975bf:JNLP4-connect connection from 10.233.108.154/10.233.108.154:58852] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4sqgt-06wl7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1b1044ea4e3811c28473390062e965308db8465a" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4sqgt" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0kc59-wf81l --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b318cd336e56da4255f1fbeb841e8ce309be6225" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0kc59" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-s1md7-9rtq8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9b24f596ebbf42381e7913140ddc40d273d22531" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-s1md7" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0dkt2-h8br0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a3ef9c6525f63aae55c9789d5f8b05ccbce3fed2" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0dkt2" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-30pk7-8xs8n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "fb88da083c4972b2f318cd837f677ebc35c30095" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-30pk7" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-fqt7j-5blb8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6030939dfc8b41ccac317e6d63fed02cabd1ef30" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-fqt7j" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-hbj6m-nh7ms --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b4485226e3090b8d616f65eecc783998173b13e3" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-hbj6m" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-t7v9z-7m56b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2e6a03a6aff3d74348a734b522a340db544a40a9" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-t7v9z" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-8wtxj-f8zbp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ac1dacdaa904e4b80c3088cc0734e8af5eafe01f" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-8wtxj" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 19.60 secs (171532473 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099 Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] stage The recommended git tool is: git [Pipeline] { (Test) [Pipeline] } [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // timeout [Pipeline] } [Pipeline] { [Pipeline] // container No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@32704824; decorates RemoteLauncher[hudson.remoting.Channel@6c0eadc4:JNLP4-connect connection from 10.233.93.171/10.233.93.171:37874] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@50295b31; decorates RemoteLauncher[hudson.remoting.Channel@7176e40d:JNLP4-connect connection from 10.233.97.21/10.233.97.21:40540] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2808e6a1; decorates RemoteLauncher[hudson.remoting.Channel@3a5006a5:JNLP4-connect connection from 10.233.126.145/10.233.126.145:51046] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@601f303d; decorates RemoteLauncher[hudson.remoting.Channel@71e0a3ac:JNLP4-connect connection from 10.233.68.145/10.233.68.145:49378] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1bd06e5a; decorates RemoteLauncher[hudson.remoting.Channel@60abde8c:JNLP4-connect connection from 10.233.84.145/10.233.84.145:57768] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_txn_v2 kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=8b92bda4-929f-4f91-a38e-4747f3b5fafb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-q7ckx GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-q7ckx ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:11:14 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@43f5b5b7; decorates RemoteLauncher[hudson.remoting.Channel@2b920a56:JNLP4-connect connection from 10.233.69.79/10.233.69.79:57408] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@423587a5; decorates RemoteLauncher[hudson.remoting.Channel@24de9461:JNLP4-connect connection from 10.233.90.83/10.233.90.83:51624] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@655728fa; decorates RemoteLauncher[hudson.remoting.Channel@2147c02c:JNLP4-connect connection from 10.233.106.210/10.233.106.210:44198] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@256c5dce; decorates RemoteLauncher[hudson.remoting.Channel@55ca1be5:JNLP4-connect connection from 10.233.88.97/10.233.88.97:55094] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-wxvfk-jrnp3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "54e2207ddcdd620d66b7a84242737e2a37b119a5" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-wxvfk" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D00B209F53DB < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:11:19 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Fri Apr 26 19:11:19 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:11:22 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:11:25 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0mโ€™ is offline Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzzโ€™ is offline Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099โ€™ is offline Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgkโ€™ is offline Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2rโ€™ is offline Still waiting to schedule task โ€˜ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjtโ€™ is offline find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:11:28 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099 is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4fsvl-d15xr Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4dcth-dcxnc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3482a04f72e59d701bebecbb1a64a7500702f234" --- apiVersion: "v1" kind: "Pod" metadata: annotations: jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4dcth" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r" namespace: "jenkins-tiflow" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" spec: affinity: nodeAffinity: runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: jenkins/label-digest: "60e3bdb269848cc6090143e14164a11dbedb71f0" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4fsvl" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: name: "workspace-volume" readOnly: false - args: - "cat" limits: cpu: "2000m" memory: "4Gi" requests: image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: cpu: "200m" memory: "4Gi" tty: true volumeMounts: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "workspace-volume" readOnly: false - args: - "cat" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" limits: cpu: "4" memory: "16Gi" requests: cpu: "2" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" - name: "KAFKA_ZOOKEEPER_CONNECT" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "RACK_COMMAND" - name: "KAFKA_SSL_KEY_PASSWORD" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ value: "test1234" \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" - name: "RACK_COMMAND" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ - mountPath: "/home/jenkins/agent" name: "workspace-volume" \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" name: "report" resources: limits: memory: "256Mi" cpu: "100m" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r" - name: "JENKINS_NAME" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" requests: memory: "256Mi" cpu: "100m" volumeMounts: value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099" - name: "JENKINS_AGENT_WORKDIR" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" - emptyDir: medium: "" name: "workspace-volume" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-tns2l-35b3t --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "52890a7a009f18d0c4504bf8acdb31987cfde99c" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-tns2l" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-49rsr-75r57 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b3515b6de86440e89573a3670d49aee51a064f89" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-49rsr" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D00E6D1347CA < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:11:33 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Fri Apr 26 19:11:33 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:11:36 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 20.58 secs (163340526 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] { [Pipeline] sh Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-gxtkz-1w600 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ce2272b662cece9efcde1065cf577b7d64f0062a" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-gxtkz" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Agent ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt is provisioned from template ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-jbhjg-bkgmj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bad47898e61c662760504f625beff0adadb19868" jenkins/label: "ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-jbhjg" name: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "4" memory: "16Gi" requests: cpu: "2" memory: "12Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt" - name: "JENKINS_NAME" value: "ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test Running on ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_txn_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@72364b59; decorates RemoteLauncher[hudson.remoting.Channel@3eb92d3e:JNLP4-connect connection from 10.233.127.209/10.233.127.209:58490] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@77d88b6c; decorates RemoteLauncher[hudson.remoting.Channel@516f6409:JNLP4-connect connection from 10.233.123.121/10.233.123.121:44926] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@26ffdff3; decorates RemoteLauncher[hudson.remoting.Channel@14640fc1:JNLP4-connect connection from 10.233.123.150/10.233.123.150:43492] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7277542d; decorates RemoteLauncher[hudson.remoting.Channel@3328d0b6:JNLP4-connect connection from 10.233.70.9/10.233.70.9:33566] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@84bc9d6; decorates RemoteLauncher[hudson.remoting.Channel@cbd7280:JNLP4-connect connection from 10.233.70.96/10.233.70.96:42662] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + grep -F -w /brokers/ids/1 + grep brokers + awk {$1=$1;print} + nc localhost 2181 /brokers/ids/1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@644d8b61; decorates RemoteLauncher[hudson.remoting.Channel@72ca2756:JNLP4-connect connection from 10.233.86.41/10.233.86.41:44646] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5848d244; decorates RemoteLauncher[hudson.remoting.Channel@74691ae:JNLP4-connect connection from 10.233.86.120/10.233.86.120:38304] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/kafka_big_txn_v2 Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 5.58 secs (601988793 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c686b15880009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:1850, start at 2024-04-26 19:11:53.20351498 +0800 CST m=+5.869345500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:53.213 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:53.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:53.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c686b15880009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:1850, start at 2024-04-26 19:11:53.20351498 +0800 CST m=+5.869345500 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:53.213 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:53.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:53.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c686b1710000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:1910, start at 2024-04-26 19:11:53.309016659 +0800 CST m=+5.819724567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:13:53.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:11:53.284 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:01:53.284 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 26 19:11:57 CST 2024] <<<<<< START cdc server in kafka_big_txn_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.31103112.out server --log-file /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 7.95 secs (422837504 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:12:00 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4718218a-f610-4b8d-9deb-b4a93edc935a {"id":"4718218a-f610-4b8d-9deb-b4a93edc935a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1a9c8af1 4718218a-f610-4b8d-9deb-b4a93edc935a /tidb/cdc/default/default/upstream/7362131881693718801 {"id":7362131881693718801,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4718218a-f610-4b8d-9deb-b4a93edc935a {"id":"4718218a-f610-4b8d-9deb-b4a93edc935a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1a9c8af1 4718218a-f610-4b8d-9deb-b4a93edc935a /tidb/cdc/default/default/upstream/7362131881693718801 {"id":7362131881693718801,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4718218a-f610-4b8d-9deb-b4a93edc935a {"id":"4718218a-f610-4b8d-9deb-b4a93edc935a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1a9c8af1 4718218a-f610-4b8d-9deb-b4a93edc935a /tidb/cdc/default/default/upstream/7362131881693718801 {"id":7362131881693718801,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.cli.3161.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348874098966529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348874098966529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:12:02 CST 2024] <<<<<< START cdc server in kafka_big_txn_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.31893191.out server --log-file /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:12:02 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4718218a-f610-4b8d-9deb-b4a93edc935a {"id":"4718218a-f610-4b8d-9deb-b4a93edc935a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1a9c8af1 4718218a-f610-4b8d-9deb-b4a93edc935a /tidb/cdc/default/default/upstream/7362131881693718801 {"id":7362131881693718801,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4718218a-f610-4b8d-9deb-b4a93edc935a {"id":"4718218a-f610-4b8d-9deb-b4a93edc935a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1a9c8af1 4718218a-f610-4b8d-9deb-b4a93edc935a /tidb/cdc/default/default/upstream/7362131881693718801 {"id":7362131881693718801,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4718218a-f610-4b8d-9deb-b4a93edc935a {"id":"4718218a-f610-4b8d-9deb-b4a93edc935a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1a9c8af1 4718218a-f610-4b8d-9deb-b4a93edc935a /tidb/cdc/default/default/upstream/7362131881693718801 {"id":7362131881693718801,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.cli.3211.out cli changefeed create --start-ts=449348874098966529 '--sink-uri=kafka://127.0.0.1:9092/ticdc-big-txn-test-31644?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_txn_v2/conf/changefeed.toml Create changefeed successfully! ID: 54f774e7-3955-41d3-a2a8-f05a49f7cd13 Info: {"upstream_id":7362131881693718801,"namespace":"default","id":"54f774e7-3955-41d3-a2a8-f05a49f7cd13","sink_uri":"kafka://127.0.0.1:9092/ticdc-big-txn-test-31644?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:12:03.423683976+08:00","start_ts":449348874098966529,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348874098966529,"checkpoint_ts":449348874098966529,"checkpoint_time":"2024-04-26 19:12:01.337"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "mysql.db"="big_txn" "fieldcount"="100" "threadcount"="1" "insertproportion"="0" "updateproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "readproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "recordcount"="5000" "operationcount"="0" "scanproportion"="0" "mysql.port"="4000" ********************************************** Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 7.74 secs (434384442 bytes/sec) [Pipeline] { [Pipeline] cache INSERT - Takes(s): 10.0, Count: 2288, OPS: 229.0, Avg(us): 4095, Min(us): 1401, Max(us): 95067, 95th(us): 4000, 99th(us): 86000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 8.91 secs (377534653 bytes/sec) [Pipeline] { [Pipeline] cache Run finished, takes 19.425934709s INSERT - Takes(s): 19.4, Count: 5000, OPS: 257.5, Avg(us): 3597, Min(us): 1312, Max(us): 95067, 95th(us): 3000, 99th(us): 80000 [Fri Apr 26 19:12:24 CST 2024] <<<<<< START kafka consumer in kafka_big_txn_v2 case >>>>>> table big_txn.usertable not exists for 1-th check, retry later table big_txn.usertable not exists for 2-th check, retry later table big_txn.usertable exists Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 7.76 secs (433155649 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 24.10 secs (139475701 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 17.69 secs (190074741 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 25.58 secs (131451795 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 26.39 secs (127394138 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + awk {$1=$1;print} + grep -F -w /brokers/ids/1 + grep brokers + sleep 10 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + + nc localhost 2181 grep brokers + grep -F -w /brokers/ids/1 + awk {$1=$1;print} + sleep 10 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh [Pipeline] } + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + grep -F -w /brokers/ids/1 + + awk {$1=$1;print} grep brokers + nc localhost 2181 /brokers/ids/1 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] sh [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G00 [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] sh Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro lossy_ddl storage_csv_update PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=d891e1c5-d9be-4353-ab9e-f0448fa4b486 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g8645 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-g8645 ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:14:31 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G01 Run cases: http_api http_api_tls api_v2 canal_json_adapter_compatibility canal_json_basic multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=3a6f863e-552c-4a24-b5d8-8971ad441e02 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0kc59 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0kc59 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:14:32 CST 2024] <<<<<< run test case http_api success! >>>>>> [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G04 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G09 Run cases: gc_safepoint changefeed_pause_resume cli savepoint synced_status PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=130a18ea-774c-4318-b880-8d72749c3967 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-30pk7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-30pk7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=b103e1d8-92e3-4f1b-8df3-8496bb1acf7b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4sqgt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4sqgt ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=1a651163-6a05-4450-9705-ae8e17b0b704 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-hbj6m GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-hbj6m ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: kafka... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G10 Run cases: default_value simple cdc_server_tips event_filter PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=0fc56549-f629-4b98-9083-153612bacf0d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-8wtxj GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-8wtxj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: kafka... <<================= table big_txn.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:14:33 CST 2024] <<<<<< run test case kafka_big_txn_v2 success! >>>>>> + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:14:35 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=3484a7c8-f5e3-4a9f-80c0-484e9fcf0dbd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-s1md7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-s1md7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=e2b6f938-5505-4e70-adaa-1e9675acae14 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-t7v9z GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-t7v9z ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 + sleep 10 start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: kafka... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:14:39 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 6.37 secs (528113079 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Verifying downstream PD is started... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: kafka... <<================= kafka downstream isn't support syncpoint record [Fri Apr 26 19:14:43 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/canal_json_adapter_compatibility Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:14:46 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875a2a8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:1153, start at 2024-04-26 19:14:46.073814402 +0800 CST m=+5.804502305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:46.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:46.058 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:46.058 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages_v2 Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875a2a8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:1153, start at 2024-04-26 19:14:46.073814402 +0800 CST m=+5.804502305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:46.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:46.058 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:46.058 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875a3680014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:1230, start at 2024-04-26 19:14:46.144806359 +0800 CST m=+5.771875442 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:46.152 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:46.156 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:46.156 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875bfb40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:1042, start at 2024-04-26 19:14:47.957411658 +0800 CST m=+5.612369158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:47.965 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:47.966 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:47.966 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:14:49 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875bfb40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:1042, start at 2024-04-26 19:14:47.957411658 +0800 CST m=+5.612369158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:47.965 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:47.966 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:47.966 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875c20c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:1114, start at 2024-04-26 19:14:48.09603734 +0800 CST m=+5.613526482 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:48.103 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:48.067 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:48.067 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... [Fri Apr 26 19:14:50 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.24712473.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Fri Apr 26 19:14:52 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.24512453.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:14:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0a380caf-90fe-47e6-9bdf-a4cdd808e278 {"id":"0a380caf-90fe-47e6-9bdf-a4cdd808e278","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d43ecf2 0a380caf-90fe-47e6-9bdf-a4cdd808e278 /tidb/cdc/default/default/upstream/7362132625104680138 {"id":7362132625104680138,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0a380caf-90fe-47e6-9bdf-a4cdd808e278 {"id":"0a380caf-90fe-47e6-9bdf-a4cdd808e278","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d43ecf2 0a380caf-90fe-47e6-9bdf-a4cdd808e278 /tidb/cdc/default/default/upstream/7362132625104680138 {"id":7362132625104680138,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0a380caf-90fe-47e6-9bdf-a4cdd808e278 {"id":"0a380caf-90fe-47e6-9bdf-a4cdd808e278","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d43ecf2 0a380caf-90fe-47e6-9bdf-a4cdd808e278 /tidb/cdc/default/default/upstream/7362132625104680138 {"id":7362132625104680138,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 4602fdf3-5aff-49eb-883d-f265338e63ef Info: {"upstream_id":7362132625104680138,"namespace":"default","id":"4602fdf3-5aff-49eb-883d-f265338e63ef","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-04-26T19:14:53.995885729+08:00","start_ts":449348918511665153,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348918511665153,"checkpoint_ts":449348918511665153,"checkpoint_time":"2024-04-26 19:14:50.758"} [Fri Apr 26 19:14:54 CST 2024] <<<<<< START kafka consumer in charset_gbk case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875ff980013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:1107, start at 2024-04-26 19:14:52.034192355 +0800 CST m=+5.706057564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:52.042 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:52.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:52.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6875ff980013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:1107, start at 2024-04-26 19:14:52.034192355 +0800 CST m=+5.706057564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:52.042 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:52.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:52.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687601e4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:1185, start at 2024-04-26 19:14:52.164800307 +0800 CST m=+5.728293025 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:52.171 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:52.153 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:52.153 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687605600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:1055, start at 2024-04-26 19:14:52.410656629 +0800 CST m=+5.880105945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:52.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:52.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:52.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687605600012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:1055, start at 2024-04-26 19:14:52.410656629 +0800 CST m=+5.880105945 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:52.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:52.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:52.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687606ec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:1130, start at 2024-04-26 19:14:52.483753048 +0800 CST m=+5.849641508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:52.493 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:52.475 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:52.475 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68762afc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:1086, start at 2024-04-26 19:14:54.795123894 +0800 CST m=+9.159291295 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:54.813 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:54.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:54.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68762afc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:1086, start at 2024-04-26 19:14:54.795123894 +0800 CST m=+9.159291295 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:54.813 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:54.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:54.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68762cc0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:1118, start at 2024-04-26 19:14:54.915255935 +0800 CST m=+9.088409949 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:54.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:54.947 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:54.947 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687631700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:1287, start at 2024-04-26 19:14:55.229599397 +0800 CST m=+5.938338085 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:55.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:55.196 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:55.196 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:14:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77697382-a671-49e6-be9e-a4eaa81f6036 {"id":"77697382-a671-49e6-be9e-a4eaa81f6036","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d441af3 77697382-a671-49e6-be9e-a4eaa81f6036 /tidb/cdc/default/default/upstream/7362132623291117130 {"id":7362132623291117130,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77697382-a671-49e6-be9e-a4eaa81f6036 {"id":"77697382-a671-49e6-be9e-a4eaa81f6036","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d441af3 77697382-a671-49e6-be9e-a4eaa81f6036 /tidb/cdc/default/default/upstream/7362132623291117130 {"id":7362132623291117130,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77697382-a671-49e6-be9e-a4eaa81f6036 {"id":"77697382-a671-49e6-be9e-a4eaa81f6036","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d441af3 77697382-a671-49e6-be9e-a4eaa81f6036 /tidb/cdc/default/default/upstream/7362132623291117130 {"id":7362132623291117130,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:14:56 CST 2024] <<<<<< START kafka consumer in gc_safepoint case >>>>>> 0 check diff failed 1-th time, retry later [Fri Apr 26 19:14:57 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.24672469.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2479.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687631700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:1287, start at 2024-04-26 19:14:55.229599397 +0800 CST m=+5.938338085 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:55.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:55.196 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:55.196 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687632280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:1358, start at 2024-04-26 19:14:55.293015001 +0800 CST m=+5.855635569 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:55.301 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:55.292 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:55.292 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2292.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449348920149540869 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348920149540869 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:14:58 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.25112513.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 14.11 secs (238346730 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.2695.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:00 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1212dd8e-34f5-41fa-b5b4-cbfca2ccf471 {"id":"1212dd8e-34f5-41fa-b5b4-cbfca2ccf471","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d5273f6 1212dd8e-34f5-41fa-b5b4-cbfca2ccf471 /tidb/cdc/default/default/upstream/7362132639228021969 {"id":7362132639228021969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1212dd8e-34f5-41fa-b5b4-cbfca2ccf471 {"id":"1212dd8e-34f5-41fa-b5b4-cbfca2ccf471","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d5273f6 1212dd8e-34f5-41fa-b5b4-cbfca2ccf471 /tidb/cdc/default/default/upstream/7362132639228021969 {"id":7362132639228021969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1212dd8e-34f5-41fa-b5b4-cbfca2ccf471 {"id":"1212dd8e-34f5-41fa-b5b4-cbfca2ccf471","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d5273f6 1212dd8e-34f5-41fa-b5b4-cbfca2ccf471 /tidb/cdc/default/default/upstream/7362132639228021969 {"id":7362132639228021969,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... + set +x + tso='449348920523620353 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + awk -F ' ' '{print $1}' + echo 449348920523620353 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + set +x [Fri Apr 26 19:15:00 CST 2024] <<<<<< START kafka consumer in kv_client_stream_reconnect case >>>>>> check diff successfully check_safepoint_forward http://127.0.0.1:2379 7362132623291117130 449348920794677252 449348919877173253 ***************** properties ***************** "recordcount"="10" "insertproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "operationcount"="0" "updateproportion"="0" "dotransactions"="false" "threadcount"="2" "readproportion"="0" "mysql.db"="sink_retry" "scanproportion"="0" "mysql.port"="4000" "workload"="core" "readallfields"="true" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 51.205701ms INSERT - Takes(s): 0.0, Count: 10, OPS: 616.6, Avg(us): 10041, Min(us): 2539, Max(us): 35342, 95th(us): 36000, 99th(us): 36000 [Fri Apr 26 19:15:01 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.23382340.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449348920946720772 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348920946720772 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:15:01 CST 2024] <<<<<< START cdc server in canal_json_adapter_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.27232725.out server --log-file /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68767018000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:4423, start at 2024-04-26 19:14:59.22737209 +0800 CST m=+5.944258683 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:59.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:59.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:59.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68767018000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:4423, start at 2024-04-26 19:14:59.22737209 +0800 CST m=+5.944258683 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:59.236 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:59.206 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:59.206 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687670e0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:4473, start at 2024-04-26 19:14:59.280456275 +0800 CST m=+5.880895918 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:16:59.289 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:14:59.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:04:59.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda {"id":"fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d55b1f7 fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda /tidb/cdc/default/default/upstream/7362132648957726178 {"id":7362132648957726178,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda {"id":"fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d55b1f7 fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda /tidb/cdc/default/default/upstream/7362132648957726178 {"id":7362132648957726178,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda {"id":"fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d55b1f7 fcdc6fd6-64dd-4b57-bbd5-f2dd80325eda /tidb/cdc/default/default/upstream/7362132648957726178 {"id":7362132648957726178,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2554.out cli changefeed create --start-ts=449348920149540869 '--sink-uri=kafka://127.0.0.1:9092/ticdc-foreign-key-test-24201?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' run task successfully Create changefeed successfully! ID: 3b7b84eb-d8c0-494c-bd1d-2a7b17871d91 Info: {"upstream_id":7362132648957726178,"namespace":"default","id":"3b7b84eb-d8c0-494c-bd1d-2a7b17871d91","sink_uri":"kafka://127.0.0.1:9092/ticdc-foreign-key-test-24201?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:02.030954375+08:00","start_ts":449348920149540869,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348920149540869,"checkpoint_ts":449348920149540869,"checkpoint_time":"2024-04-26 19:14:57.006"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 36d4c318-aac0-42cf-995f-154b33bf50ef stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=36d4c318-aac0-42cf-995f-154b33bf50ef + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 36d4c318-aac0-42cf-995f-154b33bf50ef -s + info='{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "stopped", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null }' + echo '{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "stopped", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null }' { "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "stopped", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null } ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"36d4c318-aac0-42cf-995f-154b33bf50ef",' '"state":' '"stopped",' '"checkpoint_tso":' 449348921332072450, '"checkpoint_time":' '"2024-04-26' '19:15:01.517",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"36d4c318-aac0-42cf-995f-154b33bf50ef",' '"state":' '"stopped",' '"checkpoint_tso":' 449348921332072450, '"checkpoint_time":' '"2024-04-26' '19:15:01.517",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7362132623291117130 + set +x [Fri Apr 26 19:15:03 CST 2024] <<<<<< START kafka consumer in foreign_key case >>>>>> table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:15:03 CST 2024] <<<<<< START cdc server in kafka_big_messages_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages_v2.57725774.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:04 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87ef2048-a536-4ef1-904b-a65ad376c8c7 {"id":"87ef2048-a536-4ef1-904b-a65ad376c8c7","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d52b50d 87ef2048-a536-4ef1-904b-a65ad376c8c7 /tidb/cdc/default/default/upstream/7362132643593311752 {"id":7362132643593311752,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87ef2048-a536-4ef1-904b-a65ad376c8c7 {"id":"87ef2048-a536-4ef1-904b-a65ad376c8c7","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d52b50d 87ef2048-a536-4ef1-904b-a65ad376c8c7 /tidb/cdc/default/default/upstream/7362132643593311752 {"id":7362132643593311752,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87ef2048-a536-4ef1-904b-a65ad376c8c7 {"id":"87ef2048-a536-4ef1-904b-a65ad376c8c7","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d52b50d 87ef2048-a536-4ef1-904b-a65ad376c8c7 /tidb/cdc/default/default/upstream/7362132643593311752 {"id":7362132643593311752,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2376.out cli changefeed create --start-ts=449348920523620353 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-17863?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:04 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235 {"id":"fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d6351f5 fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235 /tidb/cdc/default/default/upstream/7362132663307053264 {"id":7362132663307053264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235 {"id":"fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d6351f5 fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235 /tidb/cdc/default/default/upstream/7362132663307053264 {"id":7362132663307053264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235 {"id":"fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d6351f5 fbbeb6cb-50d1-46e9-a1e8-a6b95b34e235 /tidb/cdc/default/default/upstream/7362132663307053264 {"id":7362132663307053264,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.2771.out cli changefeed create --start-ts=449348920946720772 '--sink-uri=kafka://127.0.0.1:9092/test?protocol=canal-json&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: a1d11a7c-1ff8-40fb-8f80-f03fc97b3d4b Info: {"upstream_id":7362132663307053264,"namespace":"default","id":"a1d11a7c-1ff8-40fb-8f80-f03fc97b3d4b","sink_uri":"kafka://127.0.0.1:9092/test?protocol=canal-json\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:05.11455678+08:00","start_ts":449348920946720772,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348920946720772,"checkpoint_ts":449348920946720772,"checkpoint_time":"2024-04-26 19:15:00.047"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 2-th check, retry later run task successfully Create changefeed successfully! ID: 6556f643-4c74-4048-ac85-edd93fd29978 Info: {"upstream_id":7362132643593311752,"namespace":"default","id":"6556f643-4c74-4048-ac85-edd93fd29978","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-17863?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:05.37590301+08:00","start_ts":449348920523620353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348920523620353,"checkpoint_ts":449348920523620353,"checkpoint_time":"2024-04-26 19:14:58.433"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check_changefeed_state http://127.0.0.1:2379 36d4c318-aac0-42cf-995f-154b33bf50ef normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=36d4c318-aac0-42cf-995f-154b33bf50ef + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 36d4c318-aac0-42cf-995f-154b33bf50ef -s + info='{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "normal", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null }' + echo '{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "normal", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null }' { "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "normal", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null } ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"36d4c318-aac0-42cf-995f-154b33bf50ef",' '"state":' '"normal",' '"checkpoint_tso":' 449348921332072450, '"checkpoint_time":' '"2024-04-26' '19:15:01.517",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"36d4c318-aac0-42cf-995f-154b33bf50ef",' '"state":' '"normal",' '"checkpoint_tso":' 449348921332072450, '"checkpoint_time":' '"2024-04-26' '19:15:01.517",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7362132623291117130 449348921332072449 449348921332072450 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6876ce1c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:1132, start at 2024-04-26 19:15:05.235373047 +0800 CST m=+15.797502797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.309 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x run task successfully check_changefeed_state http://127.0.0.1:2379 36d4c318-aac0-42cf-995f-154b33bf50ef stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=36d4c318-aac0-42cf-995f-154b33bf50ef + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 36d4c318-aac0-42cf-995f-154b33bf50ef -s table test.finish_mark not exists for 3-th check, retry later + info='{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "stopped", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null }' + echo '{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "stopped", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null }' { "upstream_id": 7362132623291117130, "namespace": "default", "id": "36d4c318-aac0-42cf-995f-154b33bf50ef", "state": "stopped", "checkpoint_tso": 449348921332072450, "checkpoint_time": "2024-04-26 19:15:01.517", "error": null } ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"36d4c318-aac0-42cf-995f-154b33bf50ef",' '"state":' '"stopped",' '"checkpoint_tso":' 449348921332072450, '"checkpoint_time":' '"2024-04-26' '19:15:01.517",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"36d4c318-aac0-42cf-995f-154b33bf50ef",' '"state":' '"stopped",' '"checkpoint_tso":' 449348921332072450, '"checkpoint_time":' '"2024-04-26' '19:15:01.517",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce1acb48-7869-4f3a-8999-162fc05cd16d {"id":"ce1acb48-7869-4f3a-8999-162fc05cd16d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d70a2f3 ce1acb48-7869-4f3a-8999-162fc05cd16d /tidb/cdc/default/default/upstream/7362132676189595867 {"id":7362132676189595867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce1acb48-7869-4f3a-8999-162fc05cd16d {"id":"ce1acb48-7869-4f3a-8999-162fc05cd16d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d70a2f3 ce1acb48-7869-4f3a-8999-162fc05cd16d /tidb/cdc/default/default/upstream/7362132676189595867 {"id":7362132676189595867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce1acb48-7869-4f3a-8999-162fc05cd16d {"id":"ce1acb48-7869-4f3a-8999-162fc05cd16d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d70a2f3 ce1acb48-7869-4f3a-8999-162fc05cd16d /tidb/cdc/default/default/upstream/7362132676189595867 {"id":7362132676189595867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: cb3f2400-fb3c-47c0-b26b-b8ae83d83660 Info: {"upstream_id":7362132676189595867,"namespace":"default","id":"cb3f2400-fb3c-47c0-b26b-b8ae83d83660","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-04-26T19:15:07.099721156+08:00","start_ts":449348921919012868,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348921919012868,"checkpoint_ts":449348921919012868,"checkpoint_time":"2024-04-26 19:15:03.756"} [Fri Apr 26 19:15:07 CST 2024] <<<<<< START kafka consumer in kafka_big_messages_v2 case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 + set +x [Fri Apr 26 19:15:07 CST 2024] <<<<<< START kafka consumer in sink_retry case >>>>>> check_changefeed_state http://127.0.0.1:2379 15032e6a-5d66-41c3-8bcc-14b701570a16 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=15032e6a-5d66-41c3-8bcc-14b701570a16 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 15032e6a-5d66-41c3-8bcc-14b701570a16 -s + info='{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "15032e6a-5d66-41c3-8bcc-14b701570a16", "state": "normal", "checkpoint_tso": 449348922891829255, "checkpoint_time": "2024-04-26 19:15:07.467", "error": null }' + echo '{ "upstream_id": 7362132623291117130, "namespace": "default", "id": "15032e6a-5d66-41c3-8bcc-14b701570a16", "state": "normal", "checkpoint_tso": 449348922891829255, "checkpoint_time": "2024-04-26 19:15:07.467", "error": null }' { "upstream_id": 7362132623291117130, "namespace": "default", "id": "15032e6a-5d66-41c3-8bcc-14b701570a16", "state": "normal", "checkpoint_tso": 449348922891829255, "checkpoint_time": "2024-04-26 19:15:07.467", "error": null } ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"15032e6a-5d66-41c3-8bcc-14b701570a16",' '"state":' '"normal",' '"checkpoint_tso":' 449348922891829255, '"checkpoint_time":' '"2024-04-26' '19:15:07.467",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362132623291117130, '"namespace":' '"default",' '"id":' '"15032e6a-5d66-41c3-8bcc-14b701570a16",' '"state":' '"normal",' '"checkpoint_tso":' 449348922891829255, '"checkpoint_time":' '"2024-04-26' '19:15:07.467",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7362132623291117130 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6876ce1c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:1132, start at 2024-04-26 19:15:05.235373047 +0800 CST m=+15.797502797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:05.309 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:05.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:05.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6876bdf80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:1168, start at 2024-04-26 19:15:04.198147634 +0800 CST m=+14.427619141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:04.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:04.289 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:04.289 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2396.out cli tso query --pd=http://127.0.0.1:2379 table test.binary_columns not exists for 1-th check, retry later run task successfully Changefeed remove successfully. ID: 36d4c318-aac0-42cf-995f-154b33bf50ef CheckpointTs: 449348921332072450 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-14516?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_forward http://127.0.0.1:2379 7362132623291117130 449348922891829254 449348922891829255 449348921332072450 table test.finish_mark not exists for 5-th check, retry later table test.binary_columns not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68773070000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:1567, start at 2024-04-26 19:15:11.527719705 +0800 CST m=+5.454152779 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:11.535 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:11.516 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:11.516 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully Changefeed remove successfully. ID: 15032e6a-5d66-41c3-8bcc-14b701570a16 CheckpointTs: 449348923953512451 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-14516?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_cleared http://127.0.0.1:2379 7362132623291117130 run task successfully + set +x + tso='449348923862548486 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348923862548486 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:15:12 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + GO_FAILPOINTS= + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.24142416.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 1-th check, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68773070000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:1567, start at 2024-04-26 19:15:11.527719705 +0800 CST m=+5.454152779 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:11.535 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:11.516 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:11.516 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687730740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:1649, start at 2024-04-26 19:15:11.558224357 +0800 CST m=+5.402147030 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:11.564 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:11.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:11.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.binary_columns not exists for 3-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:15:14 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:15:15 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table kafka_big_messages.test exists check diff failed 1-th time, retry later table foreign_key.finish_mark not exists for 3-th check, retry later table test.binary_columns not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:16 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d9c23d56-64dd-4d1f-9f18-5caaf97f0e24 {"id":"d9c23d56-64dd-4d1f-9f18-5caaf97f0e24","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d5d8609 d9c23d56-64dd-4d1f-9f18-5caaf97f0e24 /tidb/cdc/default/default/upstream/7362132660602965964 {"id":7362132660602965964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d9c23d56-64dd-4d1f-9f18-5caaf97f0e24 {"id":"d9c23d56-64dd-4d1f-9f18-5caaf97f0e24","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d5d8609 d9c23d56-64dd-4d1f-9f18-5caaf97f0e24 /tidb/cdc/default/default/upstream/7362132660602965964 {"id":7362132660602965964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d9c23d56-64dd-4d1f-9f18-5caaf97f0e24 {"id":"d9c23d56-64dd-4d1f-9f18-5caaf97f0e24","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1d5d8609 d9c23d56-64dd-4d1f-9f18-5caaf97f0e24 /tidb/cdc/default/default/upstream/7362132660602965964 {"id":7362132660602965964,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2447.out cli changefeed create --start-ts=449348923862548486 '--sink-uri=kafka://127.0.0.1:9092/ticdc-default-value-test-20452?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [Fri Apr 26 19:15:16 CST 2024] <<<<<< START cdc server in kafka_big_messages case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages.28932895.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 2d08f363-e589-41e6-a7e0-f3eb0bb57d8b Info: {"upstream_id":7362132660602965964,"namespace":"default","id":"2d08f363-e589-41e6-a7e0-f3eb0bb57d8b","sink_uri":"kafka://127.0.0.1:9092/ticdc-default-value-test-20452?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:17.032749527+08:00","start_ts":449348923862548486,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348923862548486,"checkpoint_ts":449348923862548486,"checkpoint_time":"2024-04-26 19:15:11.170"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table foreign_key.finish_mark not exists for 4-th check, retry later check diff failed 2-th time, retry later table test.binary_columns not exists for 5-th check, retry later + set +x [Fri Apr 26 19:15:19 CST 2024] <<<<<< START kafka consumer in default_value case >>>>>> go: downloading github.com/google/uuid v1.6.0 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading google.golang.org/grpc v1.62.1 go: downloading golang.org/x/sync v0.7.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:19 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8ad6880-5987-4015-beb7-b2111b391e63 {"id":"c8ad6880-5987-4015-beb7-b2111b391e63","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1da088f3 c8ad6880-5987-4015-beb7-b2111b391e63 /tidb/cdc/default/default/upstream/7362132732366888110 {"id":7362132732366888110,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8ad6880-5987-4015-beb7-b2111b391e63 {"id":"c8ad6880-5987-4015-beb7-b2111b391e63","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1da088f3 c8ad6880-5987-4015-beb7-b2111b391e63 /tidb/cdc/default/default/upstream/7362132732366888110 {"id":7362132732366888110,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c8ad6880-5987-4015-beb7-b2111b391e63 {"id":"c8ad6880-5987-4015-beb7-b2111b391e63","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1da088f3 c8ad6880-5987-4015-beb7-b2111b391e63 /tidb/cdc/default/default/upstream/7362132732366888110 {"id":7362132732366888110,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff successfully go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading go.uber.org/multierr v1.11.0 Create changefeed successfully! ID: add18c93-8a0c-48cc-a698-fcd166c572e2 Info: {"upstream_id":7362132732366888110,"namespace":"default","id":"add18c93-8a0c-48cc-a698-fcd166c572e2","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-04-26T19:15:19.872639876+08:00","start_ts":449348925263708161,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348925263708161,"checkpoint_ts":449348925263708161,"checkpoint_time":"2024-04-26 19:15:16.515"} [Fri Apr 26 19:15:19 CST 2024] <<<<<< START kafka consumer in kafka_big_messages case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 table foreign_key.finish_mark not exists for 5-th check, retry later table test.binary_columns not exists for 6-th check, retry later wait process cdc.test exit for 1-th time... go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:15:21 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> go: downloading golang.org/x/text v0.14.0 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 20.33 secs (165384684 bytes/sec) [Pipeline] { [Pipeline] cache table foreign_key.finish_mark not exists for 6-th check, retry later table test.binary_columns not exists for 7-th check, retry later table test.binary_columns not exists for 8-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/google/btree v1.1.2 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading cloud.google.com/go v0.112.0 go: downloading google.golang.org/api v0.162.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 table foreign_key.finish_mark not exists for 8-th check, retry later go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/tklauser/go-sysconf v0.3.12 table kafka_big_messages.test not exists for 1-th check, retry later table test.binary_columns exists check diff failed 1-th time, retry later go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table foreign_key.finish_mark exists check diff successfully check diff successfully go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 wait process cdc.test exit for 1-th time... table kafka_big_messages.test exists check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:15:29 CST 2024] <<<<<< run test case foreign_key success! >>>>>> check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:15:32 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table test.binary_columns exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl_v2 Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "insertproportion"="0" "mysql.db"="sink_retry" "updateproportion"="0" "recordcount"="10" "dotransactions"="false" "workload"="core" "operationcount"="0" "mysql.port"="4000" "mysql.user"="root" "threadcount"="2" "readproportion"="0" "scanproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 24.424924ms INSERT - Takes(s): 0.0, Count: 10, OPS: 473.9, Avg(us): 3602, Min(us): 1409, Max(us): 9803, 95th(us): 10000, 99th(us): 10000 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 15.88 secs (211738950 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6878f41c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:4052, start at 2024-04-26 19:15:40.441378884 +0800 CST m=+5.751642503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:40.451 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:40.423 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:40.423 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:15:41 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6878fc700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:3534, start at 2024-04-26 19:15:40.997275565 +0800 CST m=+5.688223621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:41.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:41.005 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:41.005 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6878f41c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:4052, start at 2024-04-26 19:15:40.441378884 +0800 CST m=+5.751642503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:40.451 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:40.423 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:40.423 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6878f5800007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:4128, start at 2024-04-26 19:15:40.520084741 +0800 CST m=+5.714382276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:40.527 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:40.512 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:40.512 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6878fc700013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:3534, start at 2024-04-26 19:15:40.997275565 +0800 CST m=+5.688223621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:41.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:41.005 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:41.005 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6878ff640013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:3597, start at 2024-04-26 19:15:41.173148724 +0800 CST m=+5.723635091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:41.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:41.145 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:41.145 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Fri Apr 26 19:15:45 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.53305332.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 7-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:15:45 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.48634865.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/kafka_compression Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68795d1c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:7779, start at 2024-04-26 19:15:47.153210452 +0800 CST m=+5.651599952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:47.160 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:47.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:47.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:48 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12188aa5-6afb-40b9-9d0c-240f5711cb01 {"id":"12188aa5-6afb-40b9-9d0c-240f5711cb01","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e10aaf3 12188aa5-6afb-40b9-9d0c-240f5711cb01 /tidb/cdc/default/default/upstream/7362132856129729652 {"id":7362132856129729652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12188aa5-6afb-40b9-9d0c-240f5711cb01 {"id":"12188aa5-6afb-40b9-9d0c-240f5711cb01","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e10aaf3 12188aa5-6afb-40b9-9d0c-240f5711cb01 /tidb/cdc/default/default/upstream/7362132856129729652 {"id":7362132856129729652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/12188aa5-6afb-40b9-9d0c-240f5711cb01 {"id":"12188aa5-6afb-40b9-9d0c-240f5711cb01","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e10aaf3 12188aa5-6afb-40b9-9d0c-240f5711cb01 /tidb/cdc/default/default/upstream/7362132856129729652 {"id":7362132856129729652,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:15:48 CST 2024] <<<<<< START kafka consumer in changefeed_pause_resume case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table changefeed_pause_resume.t1 not exists for 1-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:48 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6de94fbd-330b-4c1c-ad70-34928569fb5d {"id":"6de94fbd-330b-4c1c-ad70-34928569fb5d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e112ff3 6de94fbd-330b-4c1c-ad70-34928569fb5d /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6de94fbd-330b-4c1c-ad70-34928569fb5d {"id":"6de94fbd-330b-4c1c-ad70-34928569fb5d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e112ff3 6de94fbd-330b-4c1c-ad70-34928569fb5d /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6de94fbd-330b-4c1c-ad70-34928569fb5d {"id":"6de94fbd-330b-4c1c-ad70-34928569fb5d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e112ff3 6de94fbd-330b-4c1c-ad70-34928569fb5d /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.4913.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7362132859119672081,"namespace":"default","id":"ddl-manager","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:49.275133359+08:00","start_ts":449348933806981125,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348933806981125,"checkpoint_ts":449348933806981125,"checkpoint_time":"2024-04-26 19:15:49.105"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68795d1c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:7779, start at 2024-04-26 19:15:47.153210452 +0800 CST m=+5.651599952 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:47.160 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:47.143 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:47.143 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68795f680014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:7852, start at 2024-04-26 19:15:47.338629667 +0800 CST m=+5.736319396 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:47.346 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:47.340 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:47.340 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 9-th time, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:15:50 CST 2024] <<<<<< START kafka consumer in ddl_manager case >>>>>> table changefeed_pause_resume.t1 not exists for 2-th check, retry later Starting Upstream TiDB... check diff failed 10-th time, retry later Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:15:52 CST 2024] <<<<<< START cdc server in multi_tables_ddl_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl_v2.90819083.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: kafka... <<================= check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 4868 exit for 1-th time... wait process 4868 exit for 2-th time... table changefeed_pause_resume.t3 exists wait process 4868 exit for 3-th time... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:15:55 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66fd8bc1-b478-4c6b-a6e4-b8a02d580efe {"id":"66fd8bc1-b478-4c6b-a6e4-b8a02d580efe","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e2b4ff4 66fd8bc1-b478-4c6b-a6e4-b8a02d580efe /tidb/cdc/default/default/upstream/7362132884855746246 {"id":7362132884855746246,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66fd8bc1-b478-4c6b-a6e4-b8a02d580efe {"id":"66fd8bc1-b478-4c6b-a6e4-b8a02d580efe","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e2b4ff4 66fd8bc1-b478-4c6b-a6e4-b8a02d580efe /tidb/cdc/default/default/upstream/7362132884855746246 {"id":7362132884855746246,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66fd8bc1-b478-4c6b-a6e4-b8a02d580efe {"id":"66fd8bc1-b478-4c6b-a6e4-b8a02d580efe","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e2b4ff4 66fd8bc1-b478-4c6b-a6e4-b8a02d580efe /tidb/cdc/default/default/upstream/7362132884855746246 {"id":7362132884855746246,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7362132884855746246,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-13668?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:55.403731114+08:00","start_ts":449348934577160193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348934577160193,"checkpoint_ts":449348934577160193,"checkpoint_time":"2024-04-26 19:15:52.043"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7362132884855746246,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-27813?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:55.595366393+08:00","start_ts":449348934577160193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348934577160193,"checkpoint_ts":449348934577160193,"checkpoint_time":"2024-04-26 19:15:52.043"} cdc.test: no process found wait process cdc.test exit for 3-th time... wait process 4868 exit for 4-th time... Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7362132884855746246,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-18165?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:15:55.789891929+08:00","start_ts":449348934577160193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348934577160193,"checkpoint_ts":449348934577160193,"checkpoint_time":"2024-04-26 19:15:52.043"} [Fri Apr 26 19:15:55 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Fri Apr 26 19:15:55 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Fri Apr 26 19:15:55 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> process cdc.test already exit [Fri Apr 26 19:15:55 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> wait process 4868 exit for 5-th time... The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 4868 exit for 6-th time... check diff failed 1-th time, retry later wait process 4868 exit for 7-th time... wait process 4868 exit for 8-th time... wait process 4868 exit for 9-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687a01780006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:5052, start at 2024-04-26 19:15:57.669814969 +0800 CST m=+5.546662724 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:57.677 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:57.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:57.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6879e6b80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:3589, start at 2024-04-26 19:15:55.967192835 +0800 CST m=+5.671825404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:55.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:55.950 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:55.950 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6879e6b80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:3589, start at 2024-04-26 19:15:55.967192835 +0800 CST m=+5.671825404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:55.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:55.950 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:55.950 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6879e83c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:3675, start at 2024-04-26 19:15:56.063386009 +0800 CST m=+5.633919580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:56.069 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:56.047 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:56.047 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (4868) - No such process wait process 4868 exit for 10-th time... process 4868 already exit [Fri Apr 26 19:15:58 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.50245026.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later table multi_tables_ddl_test.t55 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687a01780006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:5052, start at 2024-04-26 19:15:57.669814969 +0800 CST m=+5.546662724 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:57.677 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:57.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:57.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687a04500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:5139, start at 2024-04-26 19:15:57.89066649 +0800 CST m=+5.658286889 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:17:57.897 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:15:57.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:05:57.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.4957.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 3-th time, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [Fri Apr 26 19:16:02 CST 2024] <<<<<< START cdc server in kafka_compression case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.63656367.out server --log-file /tmp/tidb_cdc_test/kafka_compression/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_compression/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449348936938291201 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348936938291201 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 4-th time, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later [Fri Apr 26 19:16:03 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.50135015.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362132859119672081 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:15:49.275133359 +0800 CST StartTs:449348933806981125 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003348870 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348933846302723} {CheckpointTs:449348934226411537 MinTableBarrierTs:449348934226411537 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 {"id":"8b7c1279-5a5f-4f3e-ad1c-ffd44704a291","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e11319a 8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362132859119672081,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:15:49.275133359+08:00","start-ts":449348933806981125,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348933846302723} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348934226411537,"min-table-barrier-ts":449348934226411537,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362132859119672081 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:15:49.275133359 +0800 CST StartTs:449348933806981125 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003348870 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348933846302723} {CheckpointTs:449348934226411537 MinTableBarrierTs:449348934226411537 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 {"id":"8b7c1279-5a5f-4f3e-ad1c-ffd44704a291","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e11319a 8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362132859119672081,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:15:49.275133359+08:00","start-ts":449348933806981125,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348933846302723} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348934226411537,"min-table-barrier-ts":449348934226411537,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362132859119672081 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:15:49.275133359 +0800 CST StartTs:449348933806981125 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003348870 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348933846302723} {CheckpointTs:449348934226411537 MinTableBarrierTs:449348934226411537 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 {"id":"8b7c1279-5a5f-4f3e-ad1c-ffd44704a291","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e11319a 8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362132859119672081,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:15:49.275133359+08:00","start-ts":449348933806981125,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348933846302723} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348934226411537,"min-table-barrier-ts":449348934226411537,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 26 19:16:05 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.50855087.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:05 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28cc64fe-5e7f-4809-81f6-cd2d020a0c2a {"id":"28cc64fe-5e7f-4809-81f6-cd2d020a0c2a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e58b0f3 28cc64fe-5e7f-4809-81f6-cd2d020a0c2a /tidb/cdc/default/default/upstream/7362132934201139606 {"id":7362132934201139606,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28cc64fe-5e7f-4809-81f6-cd2d020a0c2a {"id":"28cc64fe-5e7f-4809-81f6-cd2d020a0c2a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e58b0f3 28cc64fe-5e7f-4809-81f6-cd2d020a0c2a /tidb/cdc/default/default/upstream/7362132934201139606 {"id":7362132934201139606,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28cc64fe-5e7f-4809-81f6-cd2d020a0c2a {"id":"28cc64fe-5e7f-4809-81f6-cd2d020a0c2a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e58b0f3 28cc64fe-5e7f-4809-81f6-cd2d020a0c2a /tidb/cdc/default/default/upstream/7362132934201139606 {"id":7362132934201139606,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6413.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1550c5b1-99fa-428d-9a32-6844dca524a4 {"id":"1550c5b1-99fa-428d-9a32-6844dca524a4","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e4bc20b 1550c5b1-99fa-428d-9a32-6844dca524a4 /tidb/cdc/default/default/upstream/7362132924082046031 {"id":7362132924082046031,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1550c5b1-99fa-428d-9a32-6844dca524a4 {"id":"1550c5b1-99fa-428d-9a32-6844dca524a4","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e4bc20b 1550c5b1-99fa-428d-9a32-6844dca524a4 /tidb/cdc/default/default/upstream/7362132924082046031 {"id":7362132924082046031,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1550c5b1-99fa-428d-9a32-6844dca524a4 {"id":"1550c5b1-99fa-428d-9a32-6844dca524a4","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e4bc20b 1550c5b1-99fa-428d-9a32-6844dca524a4 /tidb/cdc/default/default/upstream/7362132924082046031 {"id":7362132924082046031,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5058.out cli changefeed create --start-ts=449348936938291201 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-3733?protocol=open-protocol&partition-num=4&kafka-client-id=ddl_puller_lag&kafka-version=2.4.1&max-message-bytes=10485760' table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: 5f60db95-0ca7-48d6-8e39-da1cec538a3f Info: {"upstream_id":7362132924082046031,"namespace":"default","id":"5f60db95-0ca7-48d6-8e39-da1cec538a3f","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-3733?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=ddl_puller_lag\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:16:07.022109296+08:00","start_ts":449348936938291201,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348936938291201,"checkpoint_ts":449348936938291201,"checkpoint_time":"2024-04-26 19:16:01.050"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + tso='449348938199990277 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348938199990277 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6449.out cli changefeed create --start-ts=449348938199990277 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip' -c gzip Create changefeed successfully! ID: gzip Info: {"upstream_id":7362132934201139606,"namespace":"default","id":"gzip","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=gzip","create_time":"2024-04-26T19:16:07.701360725+08:00","start_ts":449348938199990277,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348938199990277,"checkpoint_ts":449348938199990277,"checkpoint_time":"2024-04-26 19:16:05.863"} PASS check diff successfully coverage: 2.5% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/canal_json_basic Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... + set +x [Fri Apr 26 19:16:08 CST 2024] <<<<<< START kafka consumer in ddl_puller_lag case >>>>>> < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362132859119672081 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:15:49.275133359 +0800 CST StartTs:449348933806981125 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003348870 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348933846302723} {CheckpointTs:449348934226411537 MinTableBarrierTs:449348934226411537 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 {"id":"8b7c1279-5a5f-4f3e-ad1c-ffd44704a291","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e11319a 8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362132859119672081,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:15:49.275133359+08:00","start-ts":449348933806981125,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348933846302723} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348934252888065,"min-table-barrier-ts":449348934252888065,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362132859119672081 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:15:49.275133359 +0800 CST StartTs:449348933806981125 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003348870 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348933846302723} {CheckpointTs:449348934226411537 MinTableBarrierTs:449348934226411537 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 {"id":"8b7c1279-5a5f-4f3e-ad1c-ffd44704a291","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e11319a 8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362132859119672081,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:15:49.275133359+08:00","start-ts":449348933806981125,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348933846302723} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348934252888065,"min-table-barrier-ts":449348934252888065,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362132859119672081 + grep -q 'failed to get info:' {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7362132859119672081 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:15:49.275133359 +0800 CST StartTs:449348933806981125 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003348870 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348933846302723} {CheckpointTs:449348934226411537 MinTableBarrierTs:449348934226411537 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 {"id":"8b7c1279-5a5f-4f3e-ad1c-ffd44704a291","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e11319a 8b7c1279-5a5f-4f3e-ad1c-ffd44704a291 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7362132859119672081,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-19857?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:15:49.275133359+08:00","start-ts":449348933806981125,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348933846302723} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449348934252888065,"min-table-barrier-ts":449348934252888065,"admin-job-type":0} /tidb/cdc/default/default/task/position/8b7c1279-5a5f-4f3e-ad1c-ffd44704a291/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362132859119672081 {"id":7362132859119672081,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:16:09 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:16:07.636 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:07.696 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:07.796 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:07.807 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:08.771 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:08.782 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:16:07.636 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:07.696 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:07.796 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:07.807 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:08.771 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/04/26 19:16:08.782 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]") table test.gzip_finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 25.99 secs (129351139 bytes/sec) [Pipeline] { [Pipeline] cache table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687ac554000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:3985, start at 2024-04-26 19:16:10.212194056 +0800 CST m=+5.582774873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:10.219 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:10.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:10.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348936412168205, "checkpoint_time": "2024-04-26 19:15:59.043", "error": null }' + echo '{ "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348936412168205, "checkpoint_time": "2024-04-26 19:15:59.043", "error": null }' { "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449348936412168205, "checkpoint_time": "2024-04-26 19:15:59.043", "error": null } ++ echo '{' '"upstream_id":' 7362132884855746246, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449348936412168205, '"checkpoint_time":' '"2024-04-26' '19:15:59.043",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362132884855746246, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449348936412168205, '"checkpoint_time":' '"2024-04-26' '19:15:59.043",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s table test.gzip_finish_mark not exists for 2-th check, retry later + info='{ "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348939440193540, "checkpoint_time": "2024-04-26 19:16:10.594", "error": null }' + echo '{ "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348939440193540, "checkpoint_time": "2024-04-26 19:16:10.594", "error": null }' { "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449348939440193540, "checkpoint_time": "2024-04-26 19:16:10.594", "error": null } ++ echo '{' '"upstream_id":' 7362132884855746246, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449348939440193540, '"checkpoint_time":' '"2024-04-26' '19:16:10.594",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362132884855746246, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449348939440193540, '"checkpoint_time":' '"2024-04-26' '19:16:10.594",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348935914094623, "checkpoint_time": "2024-04-26 19:15:57.143", "error": { "time": "2024-04-26T19:15:59.628452573+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:15:58.943 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449348936385953803. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348936385953803]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348935914094623, "checkpoint_time": "2024-04-26 19:15:57.143", "error": { "time": "2024-04-26T19:15:59.628452573+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:15:58.943 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449348936385953803. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348936385953803]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7362132884855746246, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449348935914094623, "checkpoint_time": "2024-04-26 19:15:57.143", "error": { "time": "2024-04-26T19:15:59.628452573+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:15:58.943 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449348936385953803. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348936385953803]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362132884855746246, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449348935914094623, '"checkpoint_time":' '"2024-04-26' '19:15:57.143",' '"error":' '{' '"time":' '"2024-04-26T19:15:59.628452573+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:15:58.943 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449348936385953803. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449348936385953803]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362132884855746246, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449348935914094623, '"checkpoint_time":' '"2024-04-26' '19:15:57.143",' '"error":' '{' '"time":' '"2024-04-26T19:15:59.628452573+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:15:58.943 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449348936385953803. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449348936385953803]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:15:58.943 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449348936385953803. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348936385953803]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:15:58.943 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449348936385953803. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449348936385953803]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully table ddl_manager.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:16:10 CST 2024] <<<<<< run test case sink_retry success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687ac554000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:3985, start at 2024-04-26 19:16:10.212194056 +0800 CST m=+5.582774873 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:10.219 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:10.197 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:10.197 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687ac6140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:4068, start at 2024-04-26 19:16:10.278923593 +0800 CST m=+5.552147815 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:10.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:10.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:10.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later table test.gzip_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6576.out cli changefeed pause -c gzip cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:16:13 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 4-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6602.out cli changefeed remove -c gzip check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:16:15 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.53095311.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: gzip CheckpointTs: 449348939078172708 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6628.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2ab87fd-03a1-4612-af83-ae1798de92ff {"id":"f2ab87fd-03a1-4612-af83-ae1798de92ff","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e8784f6 f2ab87fd-03a1-4612-af83-ae1798de92ff /tidb/cdc/default/default/upstream/7362132975849096167 {"id":7362132975849096167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2ab87fd-03a1-4612-af83-ae1798de92ff {"id":"f2ab87fd-03a1-4612-af83-ae1798de92ff","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e8784f6 f2ab87fd-03a1-4612-af83-ae1798de92ff /tidb/cdc/default/default/upstream/7362132975849096167 {"id":7362132975849096167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2ab87fd-03a1-4612-af83-ae1798de92ff {"id":"f2ab87fd-03a1-4612-af83-ae1798de92ff","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1e8784f6 f2ab87fd-03a1-4612-af83-ae1798de92ff /tidb/cdc/default/default/upstream/7362132975849096167 {"id":7362132975849096167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5355.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cdc-test-19169?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 6.08 secs (553321048 bytes/sec) [Pipeline] { [Pipeline] cache Create changefeed successfully! ID: 1f84512f-8ca4-40e0-8af1-cff739801387 Info: {"upstream_id":7362132975849096167,"namespace":"default","id":"1f84512f-8ca4-40e0-8af1-cff739801387","sink_uri":"kafka://127.0.0.1:9092/ticdc-cdc-test-19169?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:16:18.507102594+08:00","start_ts":449348941472595971,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348941472595971,"checkpoint_ts":449348941472595971,"checkpoint_time":"2024-04-26 19:16:18.347"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449348941267075077 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348941267075077 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6668.out cli changefeed create --start-ts=449348941267075077 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy' -c snappy Create changefeed successfully! ID: snappy Info: {"upstream_id":7362132934201139606,"namespace":"default","id":"snappy","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=snappy","create_time":"2024-04-26T19:16:19.432319254+08:00","start_ts":449348941267075077,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348941267075077,"checkpoint_ts":449348941267075077,"checkpoint_time":"2024-04-26 19:16:17.563"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 6-th time, retry later + set +x [Fri Apr 26 19:16:19 CST 2024] <<<<<< START kafka consumer in cdc case >>>>>> go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 table ddl_manager.finish_mark not exists for 7-th check, retry later go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 + set +x [Fri Apr 26 19:16:20 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687b49c40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:4144, start at 2024-04-26 19:16:18.709487996 +0800 CST m=+5.871047738 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:18.716 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:18.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:18.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687b49c40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:4144, start at 2024-04-26 19:16:18.709487996 +0800 CST m=+5.871047738 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:18.716 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:18.722 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:18.722 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687b4b44000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:4220, start at 2024-04-26 19:16:18.783352323 +0800 CST m=+5.848413640 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:18.790 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:18.769 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:18.769 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:16:19.389 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:19.427 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:19.524 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:19.536 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:20.524 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:20.534 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:16:19.389 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:19.427 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:19.524 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:19.536 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:20.524 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/04/26 19:16:20.534 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]") table test.snappy_finish_mark not exists for 1-th check, retry later go: downloading golang.org/x/text v0.14.0 check diff successfully table ddl_manager.finish_mark not exists for 8-th check, retry later table test.snappy_finish_mark not exists for 2-th check, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 check diff failed 1-th time, retry later [Fri Apr 26 19:16:23 CST 2024] <<<<<< START cdc server in canal_json_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.55675569.out server --log-file /tmp/tidb_cdc_test/canal_json_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_basic/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading google.golang.org/api v0.162.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 table ddl_manager.finish_mark not exists for 9-th check, retry later go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 table test.snappy_finish_mark not exists for 3-th check, retry later go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 10-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3872824e-b15f-4a5c-8f12-bf95dab8ce37 {"id":"3872824e-b15f-4a5c-8f12-bf95dab8ce37","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1ea998f1 3872824e-b15f-4a5c-8f12-bf95dab8ce37 /tidb/cdc/default/default/upstream/7362133016948095937 {"id":7362133016948095937,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3872824e-b15f-4a5c-8f12-bf95dab8ce37 {"id":"3872824e-b15f-4a5c-8f12-bf95dab8ce37","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1ea998f1 3872824e-b15f-4a5c-8f12-bf95dab8ce37 /tidb/cdc/default/default/upstream/7362133016948095937 {"id":7362133016948095937,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3872824e-b15f-4a5c-8f12-bf95dab8ce37 {"id":"3872824e-b15f-4a5c-8f12-bf95dab8ce37","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1ea998f1 3872824e-b15f-4a5c-8f12-bf95dab8ce37 /tidb/cdc/default/default/upstream/7362133016948095937 {"id":7362133016948095937,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.cli.5614.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-canal-json-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' Create changefeed successfully! ID: 90730a35-b1f9-4e42-99ef-bbda2b788ee6 Info: {"upstream_id":7362133016948095937,"namespace":"default","id":"90730a35-b1f9-4e42-99ef-bbda2b788ee6","sink_uri":"kafka://127.0.0.1:9092/ticdc-canal-json-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:16:26.742057164+08:00","start_ts":449348943628730372,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348943628730372,"checkpoint_ts":449348943628730372,"checkpoint_time":"2024-04-26 19:16:26.572"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.snappy_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6788.out cli changefeed pause -c snappy PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later + set +x start tidb cluster in /tmp/tidb_cdc_test/multi_topics_v2 Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 11-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6815.out cli changefeed remove -c snappy Changefeed remove successfully. ID: snappy CheckpointTs: 449348942158102529 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table ddl_manager.finish_mark not exists for 12-th check, retry later go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6852.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later Starting Upstream TiDB... + set +x Release Version: v7.5.1-43-ge406d5780b Edition: Community + tso='449348944845078530 PASS Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false coverage: 1.9% of statements in github.com/pingcap/tiflow/...' Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + echo 449348944845078530 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... Release Version: v7.5.1-43-ge406d5780b Edition: Community + awk -F ' ' '{print $1}' + set +x Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.6885.out cli changefeed create --start-ts=449348944845078530 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4' -c lz4 Create changefeed successfully! ID: lz4 Info: {"upstream_id":7362132934201139606,"namespace":"default","id":"lz4","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=lz4","create_time":"2024-04-26T19:16:32.982002075+08:00","start_ts":449348944845078530,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348944845078530,"checkpoint_ts":449348944845078530,"checkpoint_time":"2024-04-26 19:16:31.212"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... [Fri Apr 26 19:16:33 CST 2024] <<<<<< START kafka consumer in canal_json_basic case >>>>>> check diff successfully table ddl_manager.finish_mark not exists for 14-th check, retry later + set +x [Fri Apr 26 19:16:34 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:16:32.939 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:32.977 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:33.075 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:33.084 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:34.074 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:34.084 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:16:32.939 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:32.977 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:33.075 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:33.084 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:34.074 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/04/26 19:16:34.084 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]") table test.lz4_finish_mark not exists for 1-th check, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc) 3362002944 bytes in 15.15 secs (221934520 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { table test.lz4_finish_mark not exists for 2-th check, retry later [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] sh check diff failed 2-th time, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo+ dump nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh table ddl_manager.finish_mark not exists for 16-th check, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 table test.lz4_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7014.out cli changefeed pause -c lz4 [Pipeline] sh PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687c7dc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:10265, start at 2024-04-26 19:16:38.429285762 +0800 CST m=+5.699513497 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:38.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:38.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:38.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687c7dc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:10265, start at 2024-04-26 19:16:38.429285762 +0800 CST m=+5.699513497 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:38.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:38.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:38.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687c7f44000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6h8qw, pid:10341, start at 2024-04-26 19:16:38.494299509 +0800 CST m=+5.677601450 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:38.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:38.481 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:38.481 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7040.out cli changefeed remove -c lz4 [Pipeline] } Changefeed remove successfully. ID: lz4 CheckpointTs: 449348945697046565 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later + set +x + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7071.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.11597.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully [Pipeline] } table ddl_manager.finish_mark not exists for 19-th check, retry later + set +x + tso='449348948003913732 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348948003913732 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7098.out cli changefeed create --start-ts=449348948003913732 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd' -c zstd + set +x + tso='449348948023312385 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348948023312385 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:16:44 CST 2024] <<<<<< START cdc server in multi_topics_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.1162411626.out server --log-file /tmp/tidb_cdc_test/multi_topics_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: zstd Info: {"upstream_id":7362132934201139606,"namespace":"default","id":"zstd","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=zstd","create_time":"2024-04-26T19:16:45.035036181+08:00","start_ts":449348948003913732,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348948003913732,"checkpoint_ts":449348948003913732,"checkpoint_time":"2024-04-26 19:16:43.262"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later + set +x [Fri Apr 26 19:16:46 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/04/26 19:16:44.990 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:45.031 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:45.128 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:45.138 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:46.128 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:46.137 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]: syntax error: operand expected (error token is "[2024/04/26 19:16:44.990 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:45.031 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:45.128 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:45.138 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:46.128 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/04/26 19:16:46.137 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]") table test.zstd_finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687cf3d0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:3364, start at 2024-04-26 19:16:45.969132687 +0800 CST m=+9.250914865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:45.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:45.989 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:45.989 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687cf3d0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:3364, start at 2024-04-26 19:16:45.969132687 +0800 CST m=+9.250914865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:45.994 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:45.989 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:45.989 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687cf5580009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:3396, start at 2024-04-26 19:16:46.059493809 +0800 CST m=+9.190130017 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:18:46.076 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:16:46.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:06:46.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:47 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f83fb1c-8f48-422f-b60f-bc0a0a0e651d {"id":"5f83fb1c-8f48-422f-b60f-bc0a0a0e651d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1ef761f7 5f83fb1c-8f48-422f-b60f-bc0a0a0e651d /tidb/cdc/default/default/upstream/7362133105760327338 {"id":7362133105760327338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f83fb1c-8f48-422f-b60f-bc0a0a0e651d {"id":"5f83fb1c-8f48-422f-b60f-bc0a0a0e651d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1ef761f7 5f83fb1c-8f48-422f-b60f-bc0a0a0e651d /tidb/cdc/default/default/upstream/7362133105760327338 {"id":7362133105760327338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f83fb1c-8f48-422f-b60f-bc0a0a0e651d {"id":"5f83fb1c-8f48-422f-b60f-bc0a0a0e651d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1ef761f7 5f83fb1c-8f48-422f-b60f-bc0a0a0e651d /tidb/cdc/default/default/upstream/7362133105760327338 {"id":7362133105760327338,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.11669.out cli changefeed create --start-ts=449348948023312385 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/conf/changefeed.toml Create changefeed successfully! ID: 48ccc5f6-e14e-45bf-88f5-6f69cf6bcc5d Info: {"upstream_id":7362133105760327338,"namespace":"default","id":"48ccc5f6-e14e-45bf-88f5-6f69cf6bcc5d","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:16:48.285048558+08:00","start_ts":449348948023312385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348948023312385,"checkpoint_ts":449348948023312385,"checkpoint_time":"2024-04-26 19:16:43.336"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later table test.zstd_finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 21-th check, retry later + set +x check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later table test.zstd_finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.4612.out cli tso query --pd=http://127.0.0.1:2379 table test.zstd_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7228.out cli changefeed pause -c zstd PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 3-th check, retry later + set +x + tso='449348950357180417 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348950357180417 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 24-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7254.out cli changefeed remove -c zstd check diff failed 5-th time, retry later Changefeed remove successfully. ID: zstd CheckpointTs: 449348950376316931 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd PASS table test.finish_mark not exists for 4-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "insertproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "readproportion"="0" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "mysql.port"="4000" "mysql.db"="changefeed_error" "recordcount"="20" "workload"="core" "operationcount"="0" "scanproportion"="0" "readallfields"="true" "threadcount"="4" ********************************************** Run finished, takes 32.699544ms INSERT - Takes(s): 0.0, Count: 19, OPS: 974.1, Avg(us): 5942, Min(us): 3413, Max(us): 13061, 95th(us): 14000, 99th(us): 14000 [Fri Apr 26 19:16:55 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.46614663.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 25-th check, retry later check diff successfully + set +x wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 5-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:16:58 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:16:58 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 {"id":"a92b38f5-0d5a-4615-aad5-a2b450b5cd84","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f044115 a92b38f5-0d5a-4615-aad5-a2b450b5cd84 /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 {"id":"a92b38f5-0d5a-4615-aad5-a2b450b5cd84","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f044115 a92b38f5-0d5a-4615-aad5-a2b450b5cd84 /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 {"id":"a92b38f5-0d5a-4615-aad5-a2b450b5cd84","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f044115 a92b38f5-0d5a-4615-aad5-a2b450b5cd84 /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.4703.out cli changefeed create --start-ts=449348950357180417 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error table ddl_manager.finish_mark not exists for 26-th check, retry later Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7362133112212189461,"namespace":"default","id":"changefeed-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:16:59.043504188+08:00","start_ts":449348950357180417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348950357180417,"checkpoint_ts":449348950357180417,"checkpoint_time":"2024-04-26 19:16:52.239"} PASS table test.finish_mark not exists for 6-th check, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 27-th check, retry later + set +x [Fri Apr 26 19:17:00 CST 2024] <<<<<< START kafka consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449348950357180417, "checkpoint_time": "2024-04-26 19:16:52.239", "error": { "time": "2024-04-26T19:16:59.152085721+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348950357180117 is earlier than or equal to GC safepoint at 449348950357180417" } }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449348950357180417, "checkpoint_time": "2024-04-26 19:16:52.239", "error": { "time": "2024-04-26T19:16:59.152085721+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348950357180117 is earlier than or equal to GC safepoint at 449348950357180417" } }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449348950357180417, "checkpoint_time": "2024-04-26 19:16:52.239", "error": { "time": "2024-04-26T19:16:59.152085721+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348950357180117 is earlier than or equal to GC safepoint at 449348950357180417" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449348950357180417, '"checkpoint_time":' '"2024-04-26' '19:16:52.239",' '"error":' '{' '"time":' '"2024-04-26T19:16:59.152085721+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449348950357180117 is earlier than or equal to GC safepoint at '449348950357180417"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449348950357180417, '"checkpoint_time":' '"2024-04-26' '19:16:52.239",' '"error":' '{' '"time":' '"2024-04-26T19:16:59.152085721+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449348950357180117 is earlier than or equal to GC safepoint at '449348950357180417"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348950357180117 is earlier than or equal to GC safepoint at 449348950357180417' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449348950357180117 is earlier than or equal to GC safepoint at 449348950357180417 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.4756.out cli changefeed resume -c changefeed-error table test.finish_mark not exists for 7-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Pipeline] } [Pipeline] } [Pipeline] // timeout check diff failed 3-th time, retry later [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout table ddl_manager.finish_mark not exists for 28-th check, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G03 + set +x table changefeed_error.usertable exists table test.finish_mark not exists for 8-th check, retry later Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=011ad807-6457-4e77-946b-e84658bb23e7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0dkt2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-0dkt2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container check diff failed 1-th time, retry later [Pipeline] // timeout [Pipeline] sh [Pipeline] } [Pipeline] // container + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G08 [Pipeline] // container [Pipeline] // container table ddl_manager.finish_mark not exists for 29-th check, retry later [Pipeline] // container [Pipeline] // container [Pipeline] // container Run cases: processor_err_chan changefeed_reconstruct multi_capture PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=65031b53-1e3a-4034-963b-49dc20b5bbfd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-fqt7j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-fqt7j ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff failed 4-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=87c48fd8-4e92-4424-a4d8-4330bbfbdc20 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-wxvfk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-wxvfk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: kafka... <<================= [Pipeline] sh table test.finish_mark not exists for 9-th check, retry later [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=28de82ba-5ba9-46fa-862b-c700befbbde4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4dcth GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4dcth GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-rvx2r GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory [Pipeline] sh check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:17:06 CST 2024] <<<<<< run test case owner_resign success! >>>>>> table ddl_manager.finish_mark not exists for 30-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G14 Run cases: big_txn changefeed_finish force_replicate_table PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=297c0c4a-badf-409d-a801-0e7b6f3dabfc BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4fsvl GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099 ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-4fsvl GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/big_txn/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=c914e5f1-b98e-48c5-9b8c-7fdc039e75ae BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-tns2l GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-tns2l ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: kafka... <<================= [Pipeline] sh check diff failed 5-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=1399e81d-7e10-4f24-83a2-a1b68b48a5e7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-49rsr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-49rsr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=34eddcd4-5ecf-4464-875a-393d19c6c833 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-gxtkz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-gxtkz ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] sh table test.finish_mark not exists for 10-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=3855548d-27c4-4d0f-a096-a444ac70faae JENKINS_NODE_COOKIE=b3ec3187-cb97-4519-9aff-ef3a6cc21947 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/ GOLANG_VERSION=1.21.0 HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1783812214617018369","prowjobid":"3855548d-27c4-4d0f-a096-a444ac70faae","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#527 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783812214617018369 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/527/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-jbhjg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595 NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_527-jbhjg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-w8vjt GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=527 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:17:07 CST 2024] <<<<<< skip test case clustered_index for kafka! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:17:07 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... check diff successfully ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.port"="4000" "workload"="core" "operationcount"="0" "mysql.db"="changefeed_error" "dotransactions"="false" "readallfields"="true" "updateproportion"="0" "threadcount"="4" "requestdistribution"="uniform" "insertproportion"="0" "scanproportion"="0" "recordcount"="20" "readproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 17.418493ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1434.6, Avg(us): 2842, Min(us): 791, Max(us): 8517, 95th(us): 8000, 99th(us): 9000 check diff successfully {"id":"a92b38f5-0d5a-4615-aad5-a2b450b5cd84","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 + [[ /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later The 1 times to try to start tidb cluster... check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 31-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... table test.finish_mark not exists for 11-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:17:09 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/big_txn Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:17:09 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 32-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 + [[ /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) check diff successfully [Pipeline] // cache [Pipeline] } table test.finish_mark not exists for 12-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 33-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:17:12 CST 2024] <<<<<< run test case sink_hang success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... table test.finish_mark exists check diff successfully Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... table ddl_manager.finish_mark not exists for 34-th check, retry later TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore check diff failed 2-th time, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 + [[ /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/a92b38f5-0d5a-4615-aad5-a2b450b5cd84 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later Verifying downstream PD is started... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore [Pipeline] } [Pipeline] // timeout [Pipeline] } Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } table ddl_manager.finish_mark not exists for 35-th check, retry later [Pipeline] // podTemplate Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] } Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage check diff failed 3-th time, retry later [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 36-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 37-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_sink_error_resume Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:17:20 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.49444946.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 38-th check, retry later check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f28240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:1221, start at 2024-04-26 19:17:22.07971385 +0800 CST m=+5.586730016 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:22.087 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:22.057 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:22.057 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a36199-2a8c-4c25-b46f-de9c5aa7113d {"id":"97a36199-2a8c-4c25-b46f-de9c5aa7113d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f0441b4 97a36199-2a8c-4c25-b46f-de9c5aa7113d /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362133112212189461,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:16:59.043504188+08:00","start-ts":449348950357180417,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-04-26T19:17:21.097939696+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348957920034820} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449348954473889794,"min-table-barrier-ts":449348954473889795,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a36199-2a8c-4c25-b46f-de9c5aa7113d {"id":"97a36199-2a8c-4c25-b46f-de9c5aa7113d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f0441b4 97a36199-2a8c-4c25-b46f-de9c5aa7113d /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362133112212189461,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:16:59.043504188+08:00","start-ts":449348950357180417,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-04-26T19:17:21.097939696+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348957920034820} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449348954473889794,"min-table-barrier-ts":449348954473889795,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a36199-2a8c-4c25-b46f-de9c5aa7113d {"id":"97a36199-2a8c-4c25-b46f-de9c5aa7113d","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f0441b4 97a36199-2a8c-4c25-b46f-de9c5aa7113d /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7362133112212189461,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:16:59.043504188+08:00","start-ts":449348950357180417,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"pending","error":{"time":"2024-04-26T19:17:21.097939696+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348957920034820} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449348954473889794,"min-table-barrier-ts":449348954473889795,"admin-job-type":1} /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s table test.finish_mark not exists for 1-th check, retry later + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449348954473889794, "checkpoint_time": "2024-04-26 19:17:07.943", "error": { "time": "2024-04-26T19:17:21.097939696+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449348954473889794, "checkpoint_time": "2024-04-26 19:17:07.943", "error": { "time": "2024-04-26T19:17:21.097939696+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449348954473889794, "checkpoint_time": "2024-04-26 19:17:07.943", "error": { "time": "2024-04-26T19:17:21.097939696+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348954473889794, '"checkpoint_time":' '"2024-04-26' '19:17:07.943",' '"error":' '{' '"time":' '"2024-04-26T19:17:21.097939696+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348954473889794, '"checkpoint_time":' '"2024-04-26' '19:17:07.943",' '"error":' '{' '"time":' '"2024-04-26T19:17:21.097939696+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f2ef00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:1105, start at 2024-04-26 19:17:22.53347329 +0800 CST m=+5.927813634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:22.541 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:22.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:22.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f2ef00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:1105, start at 2024-04-26 19:17:22.53347329 +0800 CST m=+5.927813634 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:22.541 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:22.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:22.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f3088000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:1188, start at 2024-04-26 19:17:22.605718596 +0800 CST m=+5.885523112 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:22.613 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:22.594 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:22.594 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f1d640012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:1164, start at 2024-04-26 19:17:21.402286012 +0800 CST m=+5.649018867 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:21.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:21.369 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:21.369 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f1d640012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:1164, start at 2024-04-26 19:17:21.402286012 +0800 CST m=+5.649018867 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:21.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:21.369 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:21.369 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f21f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:1226, start at 2024-04-26 19:17:21.677107295 +0800 CST m=+5.825685769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:21.684 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:21.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:21.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/big_txn/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5008.out cli changefeed remove -c changefeed-error Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f3838000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:1014, start at 2024-04-26 19:17:23.119933531 +0800 CST m=+9.188811452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:23.138 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:23.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:23.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f3838000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:1014, start at 2024-04-26 19:17:23.119933531 +0800 CST m=+9.188811452 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:23.138 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:23.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:23.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f3d740002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:1049, start at 2024-04-26 19:17:23.424683864 +0800 CST m=+9.310341775 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:23.442 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:23.421 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:23.421 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 39-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f262c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:1123, start at 2024-04-26 19:17:21.942988127 +0800 CST m=+5.766470587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:21.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:21.931 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:21.931 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f262c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:1123, start at 2024-04-26 19:17:21.942988127 +0800 CST m=+5.766470587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:21.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:21.931 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:21.931 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f27ac0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:1200, start at 2024-04-26 19:17:22.02748597 +0800 CST m=+5.737492663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:22.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:22.027 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:22.027 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f28240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:1221, start at 2024-04-26 19:17:22.07971385 +0800 CST m=+5.586730016 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:22.087 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:22.057 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:22.057 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f299c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:1300, start at 2024-04-26 19:17:22.156478814 +0800 CST m=+5.523378141 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:22.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:22.151 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:22.151 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449348954473889794 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f44540009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:1075, start at 2024-04-26 19:17:23.875338594 +0800 CST m=+8.995041960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:23.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:23.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:23.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f44540009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:1075, start at 2024-04-26 19:17:23.875338594 +0800 CST m=+8.995041960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:23.891 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:23.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:23.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f49240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:1103, start at 2024-04-26 19:17:24.215565816 +0800 CST m=+9.102683585 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:24.232 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:24.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:24.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... table test.finish_mark not exists for 2-th check, retry later Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_no_changefeed 127.0.0.1:2379 table ddl_manager.finish_mark not exists for 40-th check, retry later [Fri Apr 26 19:17:26 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.24952497.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f4aa80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:1238, start at 2024-04-26 19:17:24.276570705 +0800 CST m=+5.773708819 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:24.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:24.266 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:24.266 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f4aa80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:1238, start at 2024-04-26 19:17:24.276570705 +0800 CST m=+5.773708819 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:24.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:24.266 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:24.266 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687f4b780013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:1312, start at 2024-04-26 19:17:24.35319034 +0800 CST m=+5.753692833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:24.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:24.368 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:24.368 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Fri Apr 26 19:17:26 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.24552457.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2465.out cli tso query --pd=http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 3-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2321.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2531.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table test.finish_mark not exists for 3-th check, retry later + set +x + tso='449348959425265665 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348959425265665 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:17:28 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.24912493.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:17:28 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.50715073.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449348959510986753 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348959510986753 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:17:28 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.25652567.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 41-th check, retry later + (( i++ )) + (( i <= 50 )) + set +x + tso='449348959505219585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348959505219585 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:17:28 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.23402342.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b8de0-621b-474a-a968-d7ac9dbacb52 {"id":"eb6b8de0-621b-474a-a968-d7ac9dbacb52","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9969f1 eb6b8de0-621b-474a-a968-d7ac9dbacb52 /tidb/cdc/default/default/upstream/7362133287163994019 {"id":7362133287163994019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b8de0-621b-474a-a968-d7ac9dbacb52 {"id":"eb6b8de0-621b-474a-a968-d7ac9dbacb52","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9969f1 eb6b8de0-621b-474a-a968-d7ac9dbacb52 /tidb/cdc/default/default/upstream/7362133287163994019 {"id":7362133287163994019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b8de0-621b-474a-a968-d7ac9dbacb52 {"id":"eb6b8de0-621b-474a-a968-d7ac9dbacb52","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9969f1 eb6b8de0-621b-474a-a968-d7ac9dbacb52 /tidb/cdc/default/default/upstream/7362133287163994019 {"id":7362133287163994019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:17:29 CST 2024] <<<<<< START cdc server in big_txn case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.25042506.out server --log-file /tmp/tidb_cdc_test/big_txn/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/big_txn/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b8de0-621b-474a-a968-d7ac9dbacb52 {"id":"eb6b8de0-621b-474a-a968-d7ac9dbacb52","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9969f1 eb6b8de0-621b-474a-a968-d7ac9dbacb52 /tidb/cdc/default/default/upstream/7362133287163994019 {"id":7362133287163994019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b8de0-621b-474a-a968-d7ac9dbacb52 {"id":"eb6b8de0-621b-474a-a968-d7ac9dbacb52","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9969f1 eb6b8de0-621b-474a-a968-d7ac9dbacb52 /tidb/cdc/default/default/upstream/7362133287163994019 {"id":7362133287163994019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb6b8de0-621b-474a-a968-d7ac9dbacb52 {"id":"eb6b8de0-621b-474a-a968-d7ac9dbacb52","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9969f1 eb6b8de0-621b-474a-a968-d7ac9dbacb52 /tidb/cdc/default/default/upstream/7362133287163994019 {"id":7362133287163994019,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.big_txn.cli.2530.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-big-txn-test-183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2526.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c19a846e-39ae-4dbe-b331-275bda7aa492 {"id":"c19a846e-39ae-4dbe-b331-275bda7aa492","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9cd3f0 c19a846e-39ae-4dbe-b331-275bda7aa492 /tidb/cdc/default/default/upstream/7362133295812941938 {"id":7362133295812941938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c19a846e-39ae-4dbe-b331-275bda7aa492 {"id":"c19a846e-39ae-4dbe-b331-275bda7aa492","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9cd3f0 c19a846e-39ae-4dbe-b331-275bda7aa492 /tidb/cdc/default/default/upstream/7362133295812941938 {"id":7362133295812941938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c19a846e-39ae-4dbe-b331-275bda7aa492 {"id":"c19a846e-39ae-4dbe-b331-275bda7aa492","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9cd3f0 c19a846e-39ae-4dbe-b331-275bda7aa492 /tidb/cdc/default/default/upstream/7362133295812941938 {"id":7362133295812941938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 779571c9-0590-4c6e-8014-d3e044db0c5f Info: {"upstream_id":7362133295812941938,"namespace":"default","id":"779571c9-0590-4c6e-8014-d3e044db0c5f","sink_uri":"kafka://127.0.0.1:9092/ticdc-new_ci_collation-test-14482?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:29.60150572+08:00","start_ts":449348959244648449,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348959244648449,"checkpoint_ts":449348959244648449,"checkpoint_time":"2024-04-26 19:17:26.142"} [Fri Apr 26 19:17:29 CST 2024] <<<<<< START kafka consumer in new_ci_collation case >>>>>> table test.finish_mark not exists for 4-th check, retry later Create changefeed successfully! ID: 8a81121e-36e8-4f84-9bc1-8484193427d7 Info: {"upstream_id":7362133287163994019,"namespace":"default","id":"8a81121e-36e8-4f84-9bc1-8484193427d7","sink_uri":"kafka://127.0.0.1:9092/ticdc-big-txn-test-183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:30.263108547+08:00","start_ts":449348960182337538,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348960182337538,"checkpoint_ts":449348960182337538,"checkpoint_time":"2024-04-26 19:17:29.719"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449348960037634049 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348960037634049 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:17:30 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.25612563.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 42-th check, retry later table new_ci_collation_test.t1 not exists for 1-th check, retry later check diff failed 5-th time, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7662af6-ffa9-4a60-b911-fbaaec89ef78 {"id":"b7662af6-ffa9-4a60-b911-fbaaec89ef78","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9b08f7 b7662af6-ffa9-4a60-b911-fbaaec89ef78 /tidb/cdc/default/default/upstream/7362133285316558427 {"id":7362133285316558427,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7662af6-ffa9-4a60-b911-fbaaec89ef78 {"id":"b7662af6-ffa9-4a60-b911-fbaaec89ef78","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9b08f7 b7662af6-ffa9-4a60-b911-fbaaec89ef78 /tidb/cdc/default/default/upstream/7362133285316558427 {"id":7362133285316558427,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b7662af6-ffa9-4a60-b911-fbaaec89ef78 {"id":"b7662af6-ffa9-4a60-b911-fbaaec89ef78","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f9b08f7 b7662af6-ffa9-4a60-b911-fbaaec89ef78 /tidb/cdc/default/default/upstream/7362133285316558427 {"id":7362133285316558427,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2534.out cli changefeed create --start-ts=449348959425265665 '--sink-uri=kafka://127.0.0.1:9092/ticdc-tiflash-test-8622?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 321159e2-f02c-4143-87b6-bd4176e583b4 Info: {"upstream_id":7362133285316558427,"namespace":"default","id":"321159e2-f02c-4143-87b6-bd4176e583b4","sink_uri":"kafka://127.0.0.1:9092/ticdc-tiflash-test-8622?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:31.744544337+08:00","start_ts":449348959425265665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348959425265665,"checkpoint_ts":449348959425265665,"checkpoint_time":"2024-04-26 19:17:26.831"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:31 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1910ca2-1a72-47b5-b0c8-15318ce71b4e {"id":"a1910ca2-1a72-47b5-b0c8-15318ce71b4e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f0441eb a1910ca2-1a72-47b5-b0c8-15318ce71b4e /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1910ca2-1a72-47b5-b0c8-15318ce71b4e {"id":"a1910ca2-1a72-47b5-b0c8-15318ce71b4e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f0441eb a1910ca2-1a72-47b5-b0c8-15318ce71b4e /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a1910ca2-1a72-47b5-b0c8-15318ce71b4e {"id":"a1910ca2-1a72-47b5-b0c8-15318ce71b4e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f0441eb a1910ca2-1a72-47b5-b0c8-15318ce71b4e /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5113.out cli changefeed create --start-ts=449348950357180417 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8abe8991-0e09-4222-b9ea-8d90e9251911 {"id":"8abe8991-0e09-4222-b9ea-8d90e9251911","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fa069f3 8abe8991-0e09-4222-b9ea-8d90e9251911 /tidb/cdc/default/default/upstream/7362133291850140132 {"id":7362133291850140132,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8abe8991-0e09-4222-b9ea-8d90e9251911 {"id":"8abe8991-0e09-4222-b9ea-8d90e9251911","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fa069f3 8abe8991-0e09-4222-b9ea-8d90e9251911 /tidb/cdc/default/default/upstream/7362133291850140132 {"id":7362133291850140132,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8abe8991-0e09-4222-b9ea-8d90e9251911 {"id":"8abe8991-0e09-4222-b9ea-8d90e9251911","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fa069f3 8abe8991-0e09-4222-b9ea-8d90e9251911 /tidb/cdc/default/default/upstream/7362133291850140132 {"id":7362133291850140132,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2615.out cli changefeed create --start-ts=449348959510986753 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resolve-lock-test-15120?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 397fdb5b-572a-48a8-b76f-0b9877ea3ba8 Info: {"upstream_id":7362133291850140132,"namespace":"default","id":"397fdb5b-572a-48a8-b76f-0b9877ea3ba8","sink_uri":"kafka://127.0.0.1:9092/ticdc-resolve-lock-test-15120?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:32.118040381+08:00","start_ts":449348959510986753,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348959510986753,"checkpoint_ts":449348959510986753,"checkpoint_time":"2024-04-26 19:17:27.158"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a328468-c85b-4216-aaad-339829dca6b0 {"id":"6a328468-c85b-4216-aaad-339829dca6b0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f920ef7 6a328468-c85b-4216-aaad-339829dca6b0 /tidb/cdc/default/default/upstream/7362133280809214436 {"id":7362133280809214436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a328468-c85b-4216-aaad-339829dca6b0 {"id":"6a328468-c85b-4216-aaad-339829dca6b0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f920ef7 6a328468-c85b-4216-aaad-339829dca6b0 /tidb/cdc/default/default/upstream/7362133280809214436 {"id":7362133280809214436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a328468-c85b-4216-aaad-339829dca6b0 {"id":"6a328468-c85b-4216-aaad-339829dca6b0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f920ef7 6a328468-c85b-4216-aaad-339829dca6b0 /tidb/cdc/default/default/upstream/7362133280809214436 {"id":7362133280809214436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2378.out cli changefeed create --start-ts=449348959505219585 '--sink-uri=kafka://127.0.0.1:9092/ticdc-row-format-test-26025?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7362133112212189461,"namespace":"default","id":"changefeed-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:32.392183503+08:00","start_ts":449348950357180417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348950357180417,"checkpoint_ts":449348950357180417,"checkpoint_time":"2024-04-26 19:16:52.239"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 43-th check, retry later ***************** properties ***************** "scanproportion"="0" "updateproportion"="0" "insertproportion"="0" "mysql.db"="big_txn" "requestdistribution"="uniform" "mysql.port"="4000" "readallfields"="true" "dotransactions"="false" "readproportion"="0" "operationcount"="0" "fieldcount"="100" "mysql.user"="root" "threadcount"="1" "mysql.host"="127.0.0.1" "recordcount"="5000" "workload"="core" ********************************************** table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 not exists for 1-th check, retry later Create changefeed successfully! ID: f70738f6-e965-4997-9b77-236cbdb97102 Info: {"upstream_id":7362133280809214436,"namespace":"default","id":"f70738f6-e965-4997-9b77-236cbdb97102","sink_uri":"kafka://127.0.0.1:9092/ticdc-row-format-test-26025?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:32.961143887+08:00","start_ts":449348959505219585,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348959505219585,"checkpoint_ts":449348959505219585,"checkpoint_time":"2024-04-26 19:17:27.136"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Fri Apr 26 19:17:33 CST 2024] <<<<<< START kafka consumer in tiflash case >>>>>> check diff successfully + set +x [Fri Apr 26 19:17:33 CST 2024] <<<<<< START kafka consumer in resolve_lock case >>>>>> go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 table test.finish_mark exists go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading go.uber.org/multierr v1.11.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:33 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f002a829-7bef-4f21-a625-4f5fc45f2978 {"id":"f002a829-7bef-4f21-a625-4f5fc45f2978","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fa61df5 f002a829-7bef-4f21-a625-4f5fc45f2978 /tidb/cdc/default/default/upstream/7362133293448533855 {"id":7362133293448533855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f002a829-7bef-4f21-a625-4f5fc45f2978 {"id":"f002a829-7bef-4f21-a625-4f5fc45f2978","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fa61df5 f002a829-7bef-4f21-a625-4f5fc45f2978 /tidb/cdc/default/default/upstream/7362133293448533855 {"id":7362133293448533855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f002a829-7bef-4f21-a625-4f5fc45f2978 {"id":"f002a829-7bef-4f21-a625-4f5fc45f2978","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fa61df5 f002a829-7bef-4f21-a625-4f5fc45f2978 /tidb/cdc/default/default/upstream/7362133293448533855 {"id":7362133293448533855,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2610.out cli changefeed create --start-ts=449348960037634049 '--sink-uri=kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-4363?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' check diff successfully go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 Create changefeed successfully! ID: 8bc5d126-0331-4416-9455-b8d83cbf1feb Info: {"upstream_id":7362133293448533855,"namespace":"default","id":"8bc5d126-0331-4416-9455-b8d83cbf1feb","sink_uri":"kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-4363?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:34.140100952+08:00","start_ts":449348960037634049,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348960037634049,"checkpoint_ts":449348960037634049,"checkpoint_time":"2024-04-26 19:17:29.167"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687fbe48000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:8270, start at 2024-04-26 19:17:31.683247083 +0800 CST m=+5.552803070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:31.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:31.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:31.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687fbe48000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:8270, start at 2024-04-26 19:17:31.683247083 +0800 CST m=+5.552803070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:31.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:31.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:31.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c687fbef40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:8349, start at 2024-04-26 19:17:31.750620206 +0800 CST m=+5.508404354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:19:31.757 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:17:31.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:07:31.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/beorn7/perks v1.0.1 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 table ddl_manager.finish_mark not exists for 44-th check, retry later wait process cdc.test exit for 1-th time... go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sys v0.19.0 wait process cdc.test exit for 2-th time... go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1769 0 --:--:-- --:--:-- --:--:-- 1781 + info='{"state":"warning","resolved_ts":449348950829039618,"checkpoint_ts":449348950829039618,"last_warning":{"time":"2024-04-26T19:17:34.244152439+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449348950829039618,"checkpoint_ts":449348950829039618,"last_warning":{"time":"2024-04-26T19:17:34.244152439+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449348950829039618,"checkpoint_ts":449348950829039618,"last_warning":{"time":"2024-04-26T19:17:34.244152439+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449348950829039618,"checkpoint_ts":449348950829039618,"last_warning":{"time":"2024-04-26T19:17:34.244152439+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449348950829039618,"checkpoint_ts":449348950829039618,"last_warning":{"time":"2024-04-26T19:17:34.244152439+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + set +x [Fri Apr 26 19:17:34 CST 2024] <<<<<< START kafka consumer in row_format case >>>>>> table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later check diff failed 1-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5142.out cli changefeed remove -c changefeed-error-1 wait process cdc.test exit for 3-th time... go: downloading golang.org/x/text v0.14.0 table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later + set +x [Fri Apr 26 19:17:35 CST 2024] <<<<<< START kafka consumer in many_pk_or_uk case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:17:35 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449348950829039618 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 table ddl_manager.finish_mark not exists for 45-th check, retry later [Fri Apr 26 19:17:36 CST 2024] <<<<<< START cdc server in kafka_sink_error_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_sink_error_resume.95799581.out server --log-file /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading golang.org/x/text v0.14.0 table cdc_tiflash_test.multi_data_type exists check diff successfully check diff failed 2-th time, retry later go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later + set +x wait process cdc.test exit for 1-th time... go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible [Fri Apr 26 19:17:36 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/sourcemanager/puller/ProcessorAddTableError=1*return(true)' ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.24542456.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:17:38 CST 2024] <<<<<< run test case tiflash success! >>>>>> wait process cdc.test exit for 2-th time... go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/dolthub/swiss v0.2.1 table ddl_manager.finish_mark not exists for 46-th check, retry later go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 wait process cdc.test exit for 3-th time... go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading k8s.io/apimachinery v0.28.6 table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.0 go: downloading google.golang.org/api v0.162.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b check diff failed 3-th time, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4975ce6a-19f4-46c5-9a45-b1bcbbf246af {"id":"4975ce6a-19f4-46c5-9a45-b1bcbbf246af","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f959d84 4975ce6a-19f4-46c5-9a45-b1bcbbf246af /tidb/cdc/default/default/upstream/7362133276546071808 {"id":7362133276546071808,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4975ce6a-19f4-46c5-9a45-b1bcbbf246af {"id":"4975ce6a-19f4-46c5-9a45-b1bcbbf246af","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f959d84 4975ce6a-19f4-46c5-9a45-b1bcbbf246af /tidb/cdc/default/default/upstream/7362133276546071808 {"id":7362133276546071808,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4975ce6a-19f4-46c5-9a45-b1bcbbf246af {"id":"4975ce6a-19f4-46c5-9a45-b1bcbbf246af","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f959d84 4975ce6a-19f4-46c5-9a45-b1bcbbf246af /tidb/cdc/default/default/upstream/7362133276546071808 {"id":7362133276546071808,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 26 19:17:39 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.51845186.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/fatih/color v1.16.0 [Fri Apr 26 19:17:40 CST 2024] <<<<<< START kafka consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 a5d39244-3351-4eab-9357-abb71939adbb normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=a5d39244-3351-4eab-9357-abb71939adbb + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c a5d39244-3351-4eab-9357-abb71939adbb -s + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/337ee4b4-d97f-48ab-af2a-fe417720269c {"id":"337ee4b4-d97f-48ab-af2a-fe417720269c","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fc3fef4 337ee4b4-d97f-48ab-af2a-fe417720269c /tidb/cdc/default/default/upstream/7362133336796531848 {"id":7362133336796531848,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/337ee4b4-d97f-48ab-af2a-fe417720269c {"id":"337ee4b4-d97f-48ab-af2a-fe417720269c","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fc3fef4 337ee4b4-d97f-48ab-af2a-fe417720269c /tidb/cdc/default/default/upstream/7362133336796531848 {"id":7362133336796531848,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/337ee4b4-d97f-48ab-af2a-fe417720269c {"id":"337ee4b4-d97f-48ab-af2a-fe417720269c","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1fc3fef4 337ee4b4-d97f-48ab-af2a-fe417720269c /tidb/cdc/default/default/upstream/7362133336796531848 {"id":7362133336796531848,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:17:39 CST 2024] <<<<<< START kafka consumer in kafka_sink_error_resume case >>>>>> go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/lestrrat-go/httpcc v1.0.1 + info='{ "upstream_id": 7362133276546071808, "namespace": "default", "id": "a5d39244-3351-4eab-9357-abb71939adbb", "state": "normal", "checkpoint_tso": 449348962762358787, "checkpoint_time": "2024-04-26 19:17:39.561", "error": null }' + echo '{ "upstream_id": 7362133276546071808, "namespace": "default", "id": "a5d39244-3351-4eab-9357-abb71939adbb", "state": "normal", "checkpoint_tso": 449348962762358787, "checkpoint_time": "2024-04-26 19:17:39.561", "error": null }' { "upstream_id": 7362133276546071808, "namespace": "default", "id": "a5d39244-3351-4eab-9357-abb71939adbb", "state": "normal", "checkpoint_tso": 449348962762358787, "checkpoint_time": "2024-04-26 19:17:39.561", "error": null } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362133276546071808, '"namespace":' '"default",' '"id":' '"a5d39244-3351-4eab-9357-abb71939adbb",' '"state":' '"normal",' '"checkpoint_tso":' 449348962762358787, '"checkpoint_time":' '"2024-04-26' '19:17:39.561",' '"error":' null '}' + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133276546071808, '"namespace":' '"default",' '"id":' '"a5d39244-3351-4eab-9357-abb71939adbb",' '"state":' '"normal",' '"checkpoint_tso":' 449348962762358787, '"checkpoint_time":' '"2024-04-26' '19:17:39.561",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da table ddl_manager.finish_mark not exists for 47-th check, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 check diff failed 1-th time, retry later check diff failed 4-th time, retry later go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 check_changefeed_status 127.0.0.1:8300 a2631abb-4183-48fd-8bd4-075720dad0dd warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=a2631abb-4183-48fd-8bd4-075720dad0dd + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/a2631abb-4183-48fd-8bd4-075720dad0dd/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 615 0 --:--:-- --:--:-- --:--:-- 614 100 86 100 86 0 0 615 0 --:--:-- --:--:-- --:--:-- 614 + info='{"state":"normal","resolved_ts":449348962986229794,"checkpoint_ts":449348962986229794}' + echo '{"state":"normal","resolved_ts":449348962986229794,"checkpoint_ts":449348962986229794}' {"state":"normal","resolved_ts":449348962986229794,"checkpoint_ts":449348962986229794} ++ echo '{"state":"normal","resolved_ts":449348962986229794,"checkpoint_ts":449348962986229794}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 48-th check, retry later INSERT - Takes(s): 10.0, Count: 2429, OPS: 243.1, Avg(us): 3783, Min(us): 1432, Max(us): 90972, 95th(us): 4000, 99th(us): 84000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:42 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f1d6e02-9ae4-47a6-910c-5f2216896d3e {"id":"5f1d6e02-9ae4-47a6-910c-5f2216896d3e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f044244 5f1d6e02-9ae4-47a6-910c-5f2216896d3e /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f1d6e02-9ae4-47a6-910c-5f2216896d3e {"id":"5f1d6e02-9ae4-47a6-910c-5f2216896d3e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f044244 5f1d6e02-9ae4-47a6-910c-5f2216896d3e /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f1d6e02-9ae4-47a6-910c-5f2216896d3e {"id":"5f1d6e02-9ae4-47a6-910c-5f2216896d3e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f044244 5f1d6e02-9ae4-47a6-910c-5f2216896d3e /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5220.out cli changefeed create --start-ts=449348950357180417 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-2 check diff failed 5-th time, retry later Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7362133112212189461,"namespace":"default","id":"changefeed-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:43.400377975+08:00","start_ts":449348950357180417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348950357180417,"checkpoint_ts":449348950357180417,"checkpoint_time":"2024-04-26 19:16:52.239"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check_changefeed_status 127.0.0.1:8300 a2631abb-4183-48fd-8bd4-075720dad0dd warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=a2631abb-4183-48fd-8bd4-075720dad0dd + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/a2631abb-4183-48fd-8bd4-075720dad0dd/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 check diff failed 3-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1465 0 --:--:-- --:--:-- --:--:-- 1482 check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 49-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-04-26T19:17:43.429348411+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348950357180417 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-04-26T19:17:43.429348411+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348950357180417 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 0, "checkpoint_time": "0001-01-01 00:00:00.000", "error": { "time": "2024-04-26T19:17:43.429348411+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348950357180417 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-04-26T19:17:43.429348411+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348950357180417 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 0, '"checkpoint_time":' '"0001-01-01' '00:00:00.000",' '"error":' '{' '"time":' '"2024-04-26T19:17:43.429348411+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348950357180417 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348950357180417 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449348950357180417 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5262.out cli changefeed remove -c changefeed-error-2 check diff failed 4-th time, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + info='{"state":"normal","resolved_ts":449348963720232966,"checkpoint_ts":449348963130408984}' + echo '{"state":"normal","resolved_ts":449348963720232966,"checkpoint_ts":449348963130408984}' {"state":"normal","resolved_ts":449348963720232966,"checkpoint_ts":449348963130408984} ++ echo '{"state":"normal","resolved_ts":449348963720232966,"checkpoint_ts":449348963130408984}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 2-th time, retry later check diff failed 6-th time, retry later go: downloading github.com/jmespath/go-jmespath v0.4.0 Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 0 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 50-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd wait process cdc.test exit for 2-th time... check diff successfully check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 51-th check, retry later check diff failed 1-th time, retry later + set +x wait process cdc.test exit for 1-th time... wait process cdc.test exit for 3-th time... table row_format.finish_mark not exists for 1-th check, retry later check_changefeed_status 127.0.0.1:8300 a2631abb-4183-48fd-8bd4-075720dad0dd warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=a2631abb-4183-48fd-8bd4-075720dad0dd + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/a2631abb-4183-48fd-8bd4-075720dad0dd/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2351 0 --:--:-- --:--:-- --:--:-- 2368 + info='{"state":"warning","resolved_ts":449348965306204161,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449348965306204161,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449348965306204161,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449348965306204161,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449348965306204161,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .last_warning.message + error_msg='kafka sink injected error' + [[ ! kafka sink injected error =~ kafka ]] run task successfully check diff failed 6-th time, retry later table row_format.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:17:50 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> check_changefeed_status 127.0.0.1:8300 a2631abb-4183-48fd-8bd4-075720dad0dd normal + endpoint=127.0.0.1:8300 + changefeed_id=a2631abb-4183-48fd-8bd4-075720dad0dd + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/a2631abb-4183-48fd-8bd4-075720dad0dd/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1318 0 --:--:-- --:--:-- --:--:-- 1326 + info='{"state":"warning","resolved_ts":449348965568348162,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449348965568348162,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449348965568348162,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449348965568348162,"checkpoint_ts":449348963130408984,"last_warning":{"time":"2024-04-26T19:17:43.899368575+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_topics Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 52-th check, retry later wait process cdc.test exit for 3-th time... check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:17:51 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.52995301.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully table row_format.finish_mark not exists for 3-th check, retry later check diff failed 1-th time, retry later Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 53-th check, retry later Run finished, takes 18.375448236s INSERT - Takes(s): 18.4, Count: 5000, OPS: 272.2, Avg(us): 3388, Min(us): 1379, Max(us): 90972, 95th(us): 3000, 99th(us): 79000 [Fri Apr 26 19:17:51 CST 2024] <<<<<< START kafka consumer in big_txn case >>>>>> table big_txn.usertable not exists for 1-th check, retry later check_changefeed_status 127.0.0.1:8300 a2631abb-4183-48fd-8bd4-075720dad0dd normal + endpoint=127.0.0.1:8300 + changefeed_id=a2631abb-4183-48fd-8bd4-075720dad0dd + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/a2631abb-4183-48fd-8bd4-075720dad0dd/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1368 0 --:--:-- --:--:-- --:--:-- 1387 + info='{"state":"normal","resolved_ts":449348966092636161,"checkpoint_ts":449348966092636161}' + echo '{"state":"normal","resolved_ts":449348966092636161,"checkpoint_ts":449348966092636161}' {"state":"normal","resolved_ts":449348966092636161,"checkpoint_ts":449348966092636161} ++ echo '{"state":"normal","resolved_ts":449348966092636161,"checkpoint_ts":449348966092636161}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z '' ]] ++ echo '{"state":"normal","resolved_ts":449348966092636161,"checkpoint_ts":449348966092636161}' ++ jq -r .last_error + error_msg=null + [[ ! null == \n\u\l\l ]] ++ echo '{"state":"normal","resolved_ts":449348966092636161,"checkpoint_ts":449348966092636161}' ++ jq -r .last_warning + error_msg=null + [[ ! null == \n\u\l\l ]] + exit 0 run task successfully table kafka_sink_error_resume.t1 exists table kafka_sink_error_resume.t2 exists check diff successfully check diff failed 1-th time, retry later check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... table big_txn.usertable not exists for 2-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later check diff successfully table ddl_manager.finish_mark not exists for 54-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:17:54 GMT < Content-Length: 681 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d72b4222-56a4-4d36-a8d4-cc454e57435a {"id":"d72b4222-56a4-4d36-a8d4-cc454e57435a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f04427f d72b4222-56a4-4d36-a8d4-cc454e57435a /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d72b4222-56a4-4d36-a8d4-cc454e57435a {"id":"d72b4222-56a4-4d36-a8d4-cc454e57435a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f04427f d72b4222-56a4-4d36-a8d4-cc454e57435a /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d72b4222-56a4-4d36-a8d4-cc454e57435a {"id":"d72b4222-56a4-4d36-a8d4-cc454e57435a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 6 /tidb/cdc/default/__cdc_meta__/owner/22318f1a1f04427f d72b4222-56a4-4d36-a8d4-cc454e57435a /tidb/cdc/default/default/upstream/7362133112212189461 {"id":7362133112212189461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5335.out cli changefeed create --start-ts=0 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-initialize-error wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7362133112212189461,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:17:54.969651792+08:00","start_ts":449348966741180423,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348966741180423,"checkpoint_ts":449348966741180423,"checkpoint_time":"2024-04-26 19:17:54.739"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table big_txn.usertable exists wait process cdc.test exit for 2-th time... check diff successfully wait process cdc.test exit for 1-th time... check diff failed 4-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:17:55 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> wait process cdc.test exit for 2-th time... table row_format.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 55-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:17:56 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348966741180423, "checkpoint_time": "2024-04-26 19:17:54.739", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348966741180423, "checkpoint_time": "2024-04-26 19:17:54.739", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348966741180423, "checkpoint_time": "2024-04-26 19:17:54.739", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348966741180423, '"checkpoint_time":' '"2024-04-26' '19:17:54.739",' '"error":' '{' '"time":' '"2024-04-26T19:17:55.154358791+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff failed 5-th time, retry later table row_format.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 56-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348966741180423, "checkpoint_time": "2024-04-26 19:17:54.739", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348966741180423, "checkpoint_time": "2024-04-26 19:17:54.739", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449348966741180423, "checkpoint_time": "2024-04-26 19:17:54.739", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449348966741180423, '"checkpoint_time":' '"2024-04-26' '19:17:54.739",' '"error":' '{' '"time":' '"2024-04-26T19:17:55.154358791+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later check diff failed 6-th time, retry later table row_format.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully table row_format.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 58-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:18:03 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348968890761224, "checkpoint_time": "2024-04-26 19:18:02.939", "error": null }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348968890761224, "checkpoint_time": "2024-04-26 19:18:02.939", "error": null }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348968890761224, "checkpoint_time": "2024-04-26 19:18:02.939", "error": null } ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348968890761224, '"checkpoint_time":' '"2024-04-26' '19:18:02.939",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348968890761224, '"checkpoint_time":' '"2024-04-26' '19:18:02.939",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5431.out cli changefeed pause -c changefeed-initialize-error PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 59-th check, retry later table row_format.finish_mark exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68819ac00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:6783, start at 2024-04-26 19:18:02.179742682 +0800 CST m=+5.591974416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:02.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:02.160 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:02.160 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68819ac00012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:6783, start at 2024-04-26 19:18:02.179742682 +0800 CST m=+5.591974416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:02.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:02.160 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:02.160 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68819c38000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:6862, start at 2024-04-26 19:18:02.263529804 +0800 CST m=+5.541222995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:02.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:02.254 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:02.254 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449348968890761224, "checkpoint_time": "2024-04-26 19:18:02.939", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449348968890761224, "checkpoint_time": "2024-04-26 19:18:02.939", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449348968890761224, "checkpoint_time": "2024-04-26 19:18:02.939", "error": { "time": "2024-04-26T19:17:55.154358791+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449348968890761224, '"checkpoint_time":' '"2024-04-26' '19:18:02.939",' '"error":' '{' '"time":' '"2024-04-26T19:17:55.154358791+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449348968890761224, '"checkpoint_time":' '"2024-04-26' '19:18:02.939",' '"error":' '{' '"time":' '"2024-04-26T19:17:55.154358791+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5471.out cli changefeed resume -c changefeed-initialize-error wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:18:06 CST 2024] <<<<<< run test case row_format success! >>>>>> PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 60-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.8147.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6881dfa00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:3651, start at 2024-04-26 19:18:06.609808253 +0800 CST m=+7.442996978 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:06.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:06.616 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:06.616 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6881dfa00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:3651, start at 2024-04-26 19:18:06.609808253 +0800 CST m=+7.442996978 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:06.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:06.616 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:06.616 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6881e4000011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:3733, start at 2024-04-26 19:18:06.890740212 +0800 CST m=+7.603352235 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:06.897 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:06.898 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:06.898 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348970201481224, "checkpoint_time": "2024-04-26 19:18:07.939", "error": null }' + echo '{ "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348970201481224, "checkpoint_time": "2024-04-26 19:18:07.939", "error": null }' { "upstream_id": 7362133112212189461, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449348970201481224, "checkpoint_time": "2024-04-26 19:18:07.939", "error": null } ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348970201481224, '"checkpoint_time":' '"2024-04-26' '19:18:07.939",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133112212189461, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449348970201481224, '"checkpoint_time":' '"2024-04-26' '19:18:07.939",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5512.out cli changefeed remove -c changefeed-initialize-error + set +x + tso='449348970023485441 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348970023485441 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:18:08 CST 2024] <<<<<< START cdc server in multi_topics case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.81758177.out server --log-file /tmp/tidb_cdc_test/multi_topics/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 61-th check, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449348970214588422 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-25183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Fri Apr 26 19:18:09 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.49514953.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 62-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:18:09 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> + set +x wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:11 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97c20c68-67e7-4dd3-98e2-5257520c03a5 {"id":"97c20c68-67e7-4dd3-98e2-5257520c03a5","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2038f5f7 97c20c68-67e7-4dd3-98e2-5257520c03a5 /tidb/cdc/default/default/upstream/7362133452969945667 {"id":7362133452969945667,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97c20c68-67e7-4dd3-98e2-5257520c03a5 {"id":"97c20c68-67e7-4dd3-98e2-5257520c03a5","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2038f5f7 97c20c68-67e7-4dd3-98e2-5257520c03a5 /tidb/cdc/default/default/upstream/7362133452969945667 {"id":7362133452969945667,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97c20c68-67e7-4dd3-98e2-5257520c03a5 {"id":"97c20c68-67e7-4dd3-98e2-5257520c03a5","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2038f5f7 97c20c68-67e7-4dd3-98e2-5257520c03a5 /tidb/cdc/default/default/upstream/7362133452969945667 {"id":7362133452969945667,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.8223.out cli changefeed create --start-ts=449348970023485441 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/conf/changefeed.toml =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: 89d49a59-9ee0-4b42-8470-6df8ee1ce980 Info: {"upstream_id":7362133452969945667,"namespace":"default","id":"89d49a59-9ee0-4b42-8470-6df8ee1ce980","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:18:12.238433992+08:00","start_ts":449348970023485441,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["workload.*"],"topic":"workload"},{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348970023485441,"checkpoint_ts":449348970023485441,"checkpoint_time":"2024-04-26 19:18:07.260"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table ddl_manager.finish_mark not exists for 63-th check, retry later Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:12 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b11367b-cc67-423c-b6be-66f96d91c7bd {"id":"1b11367b-cc67-423c-b6be-66f96d91c7bd","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2046cf40 1b11367b-cc67-423c-b6be-66f96d91c7bd /tidb/cdc/default/default/upstream/7362133473091435363 {"id":7362133473091435363,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b11367b-cc67-423c-b6be-66f96d91c7bd {"id":"1b11367b-cc67-423c-b6be-66f96d91c7bd","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2046cf40 1b11367b-cc67-423c-b6be-66f96d91c7bd /tidb/cdc/default/default/upstream/7362133473091435363 {"id":7362133473091435363,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b11367b-cc67-423c-b6be-66f96d91c7bd {"id":"1b11367b-cc67-423c-b6be-66f96d91c7bd","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2046cf40 1b11367b-cc67-423c-b6be-66f96d91c7bd /tidb/cdc/default/default/upstream/7362133473091435363 {"id":7362133473091435363,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.cli.5007.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-region-merge-test-22768?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' wait process cdc.test exit for 3-th time... Create changefeed successfully! ID: 082612d9-0a1d-47d4-ab77-28e2ef18ad90 Info: {"upstream_id":7362133473091435363,"namespace":"default","id":"082612d9-0a1d-47d4-ab77-28e2ef18ad90","sink_uri":"kafka://127.0.0.1:9092/ticdc-region-merge-test-22768?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:18:13.209771789+08:00","start_ts":449348971532386309,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348971532386309,"checkpoint_ts":449348971532386309,"checkpoint_time":"2024-04-26 19:18:13.016"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:18:13 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> + set +x table ddl_manager.finish_mark not exists for 64-th check, retry later + set +x [Fri Apr 26 19:18:14 CST 2024] <<<<<< START kafka consumer in region_merge case >>>>>> start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... split_and_random_merge scale: 20 *************************** 1. row *************************** count(distinct region_id): 21 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table ddl_manager.finish_mark not exists for 65-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/cli Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore table ddl_manager.finish_mark not exists for 66-th check, retry later Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/mq_sink_dispatcher Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 67-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release split_and_random_merge scale: 40 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 4-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 5-th time... wait process cdc.test exit for 6-th time... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 7-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 8-th time... process cdc.test already exit [Fri Apr 26 19:18:26 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688300e40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:3868, start at 2024-04-26 19:18:25.129340363 +0800 CST m=+5.529451001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:25.138 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:25.131 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:25.131 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688300e40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:3868, start at 2024-04-26 19:18:25.129340363 +0800 CST m=+5.529451001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:25.138 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:25.131 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:25.131 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6883026c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:3951, start at 2024-04-26 19:18:25.199027457 +0800 CST m=+5.487191187 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:25.206 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:25.179 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:25.179 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } *************************** 1. row *************************** count(distinct region_id): 1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68832e000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:3478, start at 2024-04-26 19:18:28.002368854 +0800 CST m=+9.083706493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:28.019 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:28.014 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:28.014 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68832e000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:3478, start at 2024-04-26 19:18:28.002368854 +0800 CST m=+9.083706493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:28.019 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:28.014 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:28.014 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6883339c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:3515, start at 2024-04-26 19:18:28.359681117 +0800 CST m=+9.247505525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:28.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:28.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:28.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688336500012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:9408, start at 2024-04-26 19:18:28.52647334 +0800 CST m=+5.710974075 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:28.534 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:28.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:28.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688336500012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:9408, start at 2024-04-26 19:18:28.52647334 +0800 CST m=+5.710974075 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:28.534 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:28.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:28.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688337a40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:9480, start at 2024-04-26 19:18:28.625450324 +0800 CST m=+5.721177322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:28.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:28.634 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:28.634 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Fri Apr 26 19:18:31 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.46964698.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.10740.out cli tso query --pd=http://127.0.0.1:2379 Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) split_and_random_merge scale: 80 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6883673c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:10894, start at 2024-04-26 19:18:31.669773175 +0800 CST m=+5.629476270 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:31.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:31.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:31.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6883673c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:10894, start at 2024-04-26 19:18:31.669773175 +0800 CST m=+5.629476270 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:31.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:31.681 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:31.681 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688368ac000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:10972, start at 2024-04-26 19:18:31.737699781 +0800 CST m=+5.594040589 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:31.743 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:31.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:31.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449348976901881861 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348976901881861 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4df93345-6f99-4234-bc30-5c23e94ffdfc {"id":"4df93345-6f99-4234-bc30-5c23e94ffdfc","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a208fd9f7 4df93345-6f99-4234-bc30-5c23e94ffdfc /tidb/cdc/default/default/upstream/7362133558472263915 {"id":7362133558472263915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4df93345-6f99-4234-bc30-5c23e94ffdfc {"id":"4df93345-6f99-4234-bc30-5c23e94ffdfc","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a208fd9f7 4df93345-6f99-4234-bc30-5c23e94ffdfc /tidb/cdc/default/default/upstream/7362133558472263915 {"id":7362133558472263915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4df93345-6f99-4234-bc30-5c23e94ffdfc {"id":"4df93345-6f99-4234-bc30-5c23e94ffdfc","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a208fd9f7 4df93345-6f99-4234-bc30-5c23e94ffdfc /tidb/cdc/default/default/upstream/7362133558472263915 {"id":7362133558472263915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:18:35 CST 2024] <<<<<< START kafka consumer in changefeed_reconstruct case >>>>>> ***************** properties ***************** "insertproportion"="0" "workload"="core" "readproportion"="0" "scanproportion"="0" "mysql.user"="root" "readallfields"="true" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "dotransactions"="false" "threadcount"="4" "operationcount"="0" "mysql.db"="changefeed_reconstruct" "recordcount"="50" "mysql.port"="4000" "updateproportion"="0" ********************************************** Run finished, takes 59.422502ms INSERT - Takes(s): 0.0, Count: 48, OPS: 1056.3, Avg(us): 4567, Min(us): 2007, Max(us): 14073, 95th(us): 14000, 99th(us): 15000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later [Fri Apr 26 19:18:36 CST 2024] <<<<<< START cdc server in cli case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.1079210794.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 26 19:18:35 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.52625264.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [Fri Apr 26 19:18:36 CST 2024] <<<<<< START cdc server in mq_sink_dispatcher case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.1226112263.out server --log-file /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc.log --log-level info --data-dir /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:38 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0f2da88-9ebd-485a-9cfa-6162ab7e45b0 {"id":"d0f2da88-9ebd-485a-9cfa-6162ab7e45b0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2096cd37 d0f2da88-9ebd-485a-9cfa-6162ab7e45b0 /tidb/cdc/default/default/upstream/7362133565287263104 {"id":7362133565287263104,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0f2da88-9ebd-485a-9cfa-6162ab7e45b0 {"id":"d0f2da88-9ebd-485a-9cfa-6162ab7e45b0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2096cd37 d0f2da88-9ebd-485a-9cfa-6162ab7e45b0 /tidb/cdc/default/default/upstream/7362133565287263104 {"id":7362133565287263104,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d0f2da88-9ebd-485a-9cfa-6162ab7e45b0 {"id":"d0f2da88-9ebd-485a-9cfa-6162ab7e45b0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2096cd37 d0f2da88-9ebd-485a-9cfa-6162ab7e45b0 /tidb/cdc/default/default/upstream/7362133565287263104 {"id":7362133565287263104,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5308.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-batch-add-table-test-30810?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 711cbbc2-81d3-4c82-9794-b60b10ed4449 Info: {"upstream_id":7362133565287263104,"namespace":"default","id":"711cbbc2-81d3-4c82-9794-b60b10ed4449","sink_uri":"kafka://127.0.0.1:9092/ticdc-batch-add-table-test-30810?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:18:38.635739401+08:00","start_ts":449348978208145413,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348978208145413,"checkpoint_ts":449348978208145413,"checkpoint_time":"2024-04-26 19:18:38.482"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: kafka... <<================= + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:39 GMT < Content-Length: 657 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41456d0b-22ab-44fe-aefe-9f9824cfb0bb {"id":"41456d0b-22ab-44fe-aefe-9f9824cfb0bb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20a35956 41456d0b-22ab-44fe-aefe-9f9824cfb0bb /tidb/cdc/default/default/upstream/7362133576518555571 {"id":7362133576518555571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41456d0b-22ab-44fe-aefe-9f9824cfb0bb {"id":"41456d0b-22ab-44fe-aefe-9f9824cfb0bb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20a35956 41456d0b-22ab-44fe-aefe-9f9824cfb0bb /tidb/cdc/default/default/upstream/7362133576518555571 {"id":7362133576518555571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41456d0b-22ab-44fe-aefe-9f9824cfb0bb {"id":"41456d0b-22ab-44fe-aefe-9f9824cfb0bb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20a35956 41456d0b-22ab-44fe-aefe-9f9824cfb0bb /tidb/cdc/default/default/upstream/7362133576518555571 {"id":7362133576518555571,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.10841.out cli changefeed create --start-ts=449348976901881861 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-5285?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7362133576518555571,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-5285?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:18:39.535614122+08:00","start_ts":449348976901881861,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348976901881861,"checkpoint_ts":449348976901881861,"checkpoint_time":"2024-04-26 19:18:33.499"} PASS < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:39 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bb3fce38-ebbc-4872-af3a-2b735e0ac932 {"id":"bb3fce38-ebbc-4872-af3a-2b735e0ac932","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20ac14f2 bb3fce38-ebbc-4872-af3a-2b735e0ac932 /tidb/cdc/default/default/upstream/7362133590076406962 {"id":7362133590076406962,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bb3fce38-ebbc-4872-af3a-2b735e0ac932 {"id":"bb3fce38-ebbc-4872-af3a-2b735e0ac932","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20ac14f2 bb3fce38-ebbc-4872-af3a-2b735e0ac932 /tidb/cdc/default/default/upstream/7362133590076406962 {"id":7362133590076406962,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bb3fce38-ebbc-4872-af3a-2b735e0ac932 {"id":"bb3fce38-ebbc-4872-af3a-2b735e0ac932","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20ac14f2 bb3fce38-ebbc-4872-af3a-2b735e0ac932 /tidb/cdc/default/default/upstream/7362133590076406962 {"id":7362133590076406962,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... *************************** 1. row *************************** count(distinct region_id): 1 + set +x [Fri Apr 26 19:18:40 CST 2024] <<<<<< START kafka consumer in batch_add_table case >>>>>> TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later table batch_add_table.finish_mark not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12331.out cli tso query --pd=http://127.0.0.1:2379 + set +x [Fri Apr 26 19:18:40 CST 2024] <<<<<< START kafka consumer in cli case >>>>>> table test.simple not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + set +x + tso='449348978837291010 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348978837291010 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12362.out cli changefeed create --start-ts=449348978837291010 '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/changefeed.toml Create changefeed successfully! ID: test Info: {"upstream_id":7362133590076406962,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:18:42.676929502+08:00","start_ts":449348978837291010,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["verify.t"],"partition":"index-value"},{"matcher":["dispatcher.index"],"partition":"index-value","index":"idx_a"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348978837291010,"checkpoint_ts":449348978837291010,"checkpoint_time":"2024-04-26 19:18:40.882"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process 4701 exit for 1-th time... table test.simple not exists for 2-th check, retry later table batch_add_table.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 4701 exit for 2-th time... wait process cdc.test exit for 2-th time... wait process 4701 exit for 3-th time... + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s table region_merge.t1 exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348978837291010, "checkpoint_time": "2024-04-26 19:18:40.882", "error": null }' + echo '{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348978837291010, "checkpoint_time": "2024-04-26 19:18:40.882", "error": null }' { "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348978837291010, "checkpoint_time": "2024-04-26 19:18:40.882", "error": null } ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449348978837291010, '"checkpoint_time":' '"2024-04-26' '19:18:40.882",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449348978837291010, '"checkpoint_time":' '"2024-04-26' '19:18:40.882",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:18:44 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (4701) - No such process wait process 4701 exit for 4-th time... process 4701 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 26 19:18:44 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.48844886.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688426900002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:3355, start at 2024-04-26 19:18:43.879348477 +0800 CST m=+10.845916997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:43.896 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:43.876 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:43.876 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688426900002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:3355, start at 2024-04-26 19:18:43.879348477 +0800 CST m=+10.845916997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:43.896 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:43.876 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:43.876 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6884120c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:3379, start at 2024-04-26 19:18:42.604216491 +0800 CST m=+9.439539459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:42.683 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:42.713 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:42.713 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348977413324835, "checkpoint_time": "2024-04-26 19:18:35.450", "error": null }' + echo '{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348977413324835, "checkpoint_time": "2024-04-26 19:18:35.450", "error": null }' { "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348977413324835, "checkpoint_time": "2024-04-26 19:18:35.450", "error": null } ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348977413324835, '"checkpoint_time":' '"2024-04-26' '19:18:35.450",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348977413324835, '"checkpoint_time":' '"2024-04-26' '19:18:35.450",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348978837291010, "checkpoint_time": "2024-04-26 19:18:40.882", "error": null }' + echo '{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348978837291010, "checkpoint_time": "2024-04-26 19:18:40.882", "error": null }' { "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348978837291010, "checkpoint_time": "2024-04-26 19:18:40.882", "error": null } ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449348978837291010, '"checkpoint_time":' '"2024-04-26' '19:18:40.882",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 1-th time, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc002e22ac8} {Type:update Path:[Config SyncPointRetention] From: To:0xc002e22ae0} {Type:update Path:[Config Consistent] From: To:0xc0003382a0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11112.out cli changefeed --changefeed-id custom-changefeed-name pause wait process cdc.test exit for 2-th time... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:18:47 CST 2024] <<<<<< run test case region_merge success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/070d7d9c-2b32-4934-940d-c49e1ae63e19 {UpstreamID:7362133558472263915 Namespace:default ID:070d7d9c-2b32-4934-940d-c49e1ae63e19 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-26059?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:18:35.201196161 +0800 CST StartTs:449348977272553473 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001565cb0 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348977338351621} {CheckpointTs:449348980497186817 MinTableBarrierTs:449348980497186817 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449348980497186817, checkpointTs: 449348980497186817, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/035e5dd2-7027-46be-8a32-bcc6f8e515bf {"id":"035e5dd2-7027-46be-8a32-bcc6f8e515bf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a208fda76 035e5dd2-7027-46be-8a32-bcc6f8e515bf /tidb/cdc/default/default/changefeed/info/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"upstream-id":7362133558472263915,"namespace":"default","changefeed-id":"070d7d9c-2b32-4934-940d-c49e1ae63e19","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-26059?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:18:35.201196161+08:00","start-ts":449348977272553473,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348977338351621} /tidb/cdc/default/default/changefeed/status/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"checkpoint-ts":449348980497186817,"min-table-barrier-ts":449348980497186817,"admin-job-type":0} /tidb/cdc/default/default/task/position/035e5dd2-7027-46be-8a32-bcc6f8e515bf/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133558472263915 {"id":7362133558472263915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/070d7d9c-2b32-4934-940d-c49e1ae63e19 {UpstreamID:7362133558472263915 Namespace:default ID:070d7d9c-2b32-4934-940d-c49e1ae63e19 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-26059?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:18:35.201196161 +0800 CST StartTs:449348977272553473 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001565cb0 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348977338351621} {CheckpointTs:449348980497186817 MinTableBarrierTs:449348980497186817 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449348980497186817, checkpointTs: 449348980497186817, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/035e5dd2-7027-46be-8a32-bcc6f8e515bf {"id":"035e5dd2-7027-46be-8a32-bcc6f8e515bf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a208fda76 035e5dd2-7027-46be-8a32-bcc6f8e515bf /tidb/cdc/default/default/changefeed/info/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"upstream-id":7362133558472263915,"namespace":"default","changefeed-id":"070d7d9c-2b32-4934-940d-c49e1ae63e19","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-26059?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:18:35.201196161+08:00","start-ts":449348977272553473,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348977338351621} /tidb/cdc/default/default/changefeed/status/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"checkpoint-ts":449348980497186817,"min-table-barrier-ts":449348980497186817,"admin-job-type":0} /tidb/cdc/default/default/task/position/035e5dd2-7027-46be-8a32-bcc6f8e515bf/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133558472263915 {"id":7362133558472263915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/070d7d9c-2b32-4934-940d-c49e1ae63e19 {UpstreamID:7362133558472263915 Namespace:default ID:070d7d9c-2b32-4934-940d-c49e1ae63e19 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-26059?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:18:35.201196161 +0800 CST StartTs:449348977272553473 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001565cb0 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348977338351621} {CheckpointTs:449348980497186817 MinTableBarrierTs:449348980497186817 AdminJobType:noop} span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449348980497186817, checkpointTs: 449348980497186817, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/035e5dd2-7027-46be-8a32-bcc6f8e515bf {"id":"035e5dd2-7027-46be-8a32-bcc6f8e515bf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a208fda76 035e5dd2-7027-46be-8a32-bcc6f8e515bf /tidb/cdc/default/default/changefeed/info/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"upstream-id":7362133558472263915,"namespace":"default","changefeed-id":"070d7d9c-2b32-4934-940d-c49e1ae63e19","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-26059?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:18:35.201196161+08:00","start-ts":449348977272553473,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348977338351621} /tidb/cdc/default/default/changefeed/status/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"checkpoint-ts":449348980497186817,"min-table-barrier-ts":449348980497186817,"admin-job-type":0} /tidb/cdc/default/default/task/position/035e5dd2-7027-46be-8a32-bcc6f8e515bf/070d7d9c-2b32-4934-940d-c49e1ae63e19 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133558472263915 {"id":7362133558472263915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id + info='{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348979741425691, "checkpoint_time": "2024-04-26 19:18:44.331", "error": null }' + echo '{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348979741425691, "checkpoint_time": "2024-04-26 19:18:44.331", "error": null }' { "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348979741425691, "checkpoint_time": "2024-04-26 19:18:44.331", "error": null } ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449348979741425691, '"checkpoint_time":' '"2024-04-26' '19:18:44.331",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 2-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.4594.out cli tso query --pd=http://127.0.0.1:2379 + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release "id": "035e5dd2-7027-46be-8a32-bcc6f8e515bf", "cluster-id": "default" run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449348980749369345 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348980749369345 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:18:49 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.46184620.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: 035e5dd2-7027-46be-8a32-bcc6f8e515bf check_processor_table_count http://127.0.0.1:2379 070d7d9c-2b32-4934-940d-c49e1ae63e19 035e5dd2-7027-46be-8a32-bcc6f8e515bf 1 run task successfully check_processor_table_count http://127.0.0.1:2379 070d7d9c-2b32-4934-940d-c49e1ae63e19 035e5dd2-7027-46be-8a32-bcc6f8e515bf 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68848a80000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:6345, start at 2024-04-26 19:18:50.307286035 +0800 CST m=+8.313983574 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:50.325 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:50.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:50.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68848a80000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:6345, start at 2024-04-26 19:18:50.307286035 +0800 CST m=+8.313983574 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:50.325 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:50.322 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:50.322 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449348980152729606, "checkpoint_time": "2024-04-26 19:18:45.900", "error": null }' + echo '{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449348980152729606, "checkpoint_time": "2024-04-26 19:18:45.900", "error": null }' { "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449348980152729606, "checkpoint_time": "2024-04-26 19:18:45.900", "error": null } ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449348980152729606, '"checkpoint_time":' '"2024-04-26' '19:18:45.900",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449348980152729606, '"checkpoint_time":' '"2024-04-26' '19:18:45.900",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11177.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name table count 1 does equal to expected count 0 run task failed 1-th time, retry later check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449348979885604865, "checkpoint_time": "2024-04-26 19:18:44.881", "error": { "time": "2024-04-26T19:18:49.420064628+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a" } }' + echo '{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449348979885604865, "checkpoint_time": "2024-04-26 19:18:44.881", "error": { "time": "2024-04-26T19:18:49.420064628+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a" } }' { "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449348979885604865, "checkpoint_time": "2024-04-26 19:18:44.881", "error": { "time": "2024-04-26T19:18:49.420064628+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449348979885604865, '"checkpoint_time":' '"2024-04-26' '19:18:44.881",' '"error":' '{' '"time":' '"2024-04-26T19:18:49.420064628+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: dispatcher.index, index: 'idx_a"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449348979885604865, '"checkpoint_time":' '"2024-04-26' '19:18:44.881",' '"error":' '{' '"time":' '"2024-04-26T19:18:49.420064628+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: dispatcher.index, index: 'idx_a"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a' + [[ ! [CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a =~ ErrDispatcherFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12569.out cli changefeed update -c test '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/new_changefeed.toml --no-confirm Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc00309c338} {Type:update Path:[Config SyncPointRetention] From: To:0xc00309c348} {Type:update Path:[Config Consistent] From: To:0xc0003a8150} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7362133576518555571,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-5285?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:18:39.535614122+08:00","start_ts":449348976901881861,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348980414873608,"checkpoint_ts":449348980152729606,"checkpoint_time":"2024-04-26 19:18:45.900"} PASS Diff of changefeed config: {Type:update Path:[Config SyncPointInterval] From: To:0xc002c7c768} {Type:update Path:[Config SyncPointRetention] From: To:0xc002c7c778} {Type:update Path:[Config Sink DispatchRules 0 Matcher 0] From:verify.t To:dispatcher.index} {Type:delete Path:[Config Sink DispatchRules 1 Matcher 0] From:dispatcher.index To:} {Type:delete Path:[Config Sink DispatchRules 1 PartitionRule] From:index-value To:} {Type:delete Path:[Config Sink DispatchRules 1 IndexName] From:idx_a To:} {Type:update Path:[Config Consistent] From: To:0xc0012db260} coverage: 2.9% of statements in github.com/pingcap/tiflow/... Update changefeed config successfully! ID: test Info: {"upstream_id":7362133590076406962,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:18:42.676929502+08:00","start_ts":449348978837291010,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","dispatchers":[{"matcher":["dispatcher.index"],"partition":"index-value"}],"encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"failed","error":{"addr":"127.0.0.1:8300","code":"CDC:ErrDispatcherFailed","message":"[CDC:ErrDispatcherFailed]index not found when dispatch event, table: dispatcher.index, index: idx_a"},"creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348980999716870,"checkpoint_ts":449348979885604865,"checkpoint_time":"2024-04-26 19:18:44.881"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:18:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cdc7c89-0f42-4065-b125-fbccec29f3e9 {"id":"0cdc7c89-0f42-4065-b125-fbccec29f3e9","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20c8befa 0cdc7c89-0f42-4065-b125-fbccec29f3e9 /tidb/cdc/default/default/upstream/7362133620501698612 {"id":7362133620501698612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cdc7c89-0f42-4065-b125-fbccec29f3e9 {"id":"0cdc7c89-0f42-4065-b125-fbccec29f3e9","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20c8befa 0cdc7c89-0f42-4065-b125-fbccec29f3e9 /tidb/cdc/default/default/upstream/7362133620501698612 {"id":7362133620501698612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0cdc7c89-0f42-4065-b125-fbccec29f3e9 {"id":"0cdc7c89-0f42-4065-b125-fbccec29f3e9","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20c8befa 0cdc7c89-0f42-4065-b125-fbccec29f3e9 /tidb/cdc/default/default/upstream/7362133620501698612 {"id":7362133620501698612,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.4658.out cli changefeed create --start-ts=449348980749369345 '--sink-uri=kafka://127.0.0.1:9092/ticdc-drop-tables-test-11267?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68849bc80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:6382, start at 2024-04-26 19:18:51.38776182 +0800 CST m=+9.226838902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:51.406 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:51.378 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:51.378 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11212.out cli changefeed --changefeed-id custom-changefeed-name resume + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.12600.out cli changefeed resume -c test check_processor_table_count http://127.0.0.1:2379 070d7d9c-2b32-4934-940d-c49e1ae63e19 035e5dd2-7027-46be-8a32-bcc6f8e515bf 0 run task successfully PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "workload"="core" "mysql.db"="changefeed_reconstruct" "mysql.user"="root" "recordcount"="50" "scanproportion"="0" "insertproportion"="0" "threadcount"="4" "readproportion"="0" "readallfields"="true" "mysql.port"="4000" "operationcount"="0" "requestdistribution"="uniform" "dotransactions"="false" "updateproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 38.100334ms INSERT - Takes(s): 0.0, Count: 48, OPS: 1668.4, Avg(us): 2983, Min(us): 1774, Max(us): 9783, 95th(us): 10000, 99th(us): 10000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.7568.out cli tso query --pd=http://127.0.0.1:2379 + set +x [Fri Apr 26 19:18:51 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 1 [Fri Apr 26 19:18:51 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 2 [Fri Apr 26 19:18:51 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 3 table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later table changefeed_reconstruct.usertable exists + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348979885604865, "checkpoint_time": "2024-04-26 19:18:44.881", "error": null }' + echo '{ "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348979885604865, "checkpoint_time": "2024-04-26 19:18:44.881", "error": null }' { "upstream_id": 7362133590076406962, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449348979885604865, "checkpoint_time": "2024-04-26 19:18:44.881", "error": null } ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449348979885604865, '"checkpoint_time":' '"2024-04-26' '19:18:44.881",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133590076406962, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449348979885604865, '"checkpoint_time":' '"2024-04-26' '19:18:44.881",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Create changefeed successfully! ID: ce28287b-340a-4a2f-b0c7-e64830fc4df1 Info: {"upstream_id":7362133620501698612,"namespace":"default","id":"ce28287b-340a-4a2f-b0c7-e64830fc4df1","sink_uri":"kafka://127.0.0.1:9092/ticdc-drop-tables-test-11267?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:18:56.061139912+08:00","start_ts":449348980749369345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348980749369345,"checkpoint_ts":449348980749369345,"checkpoint_time":"2024-04-26 19:18:48.176"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table test.finish_mark not exists for 1-th check, retry later + set +x [Fri Apr 26 19:18:57 CST 2024] <<<<<< START kafka consumer in drop_many_tables case >>>>>> table test.table10 exists table test.table20 exists check diff successfully + set +x + tso='449348982858579975 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348982858579975 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:18:57 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.75887590.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6884e59c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:6421, start at 2024-04-26 19:18:56.152515631 +0800 CST m=+5.780141385 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:56.161 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:56.153 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:56.153 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6884e59c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:6421, start at 2024-04-26 19:18:56.152515631 +0800 CST m=+5.780141385 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:56.161 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:56.153 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:56.153 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6884e7080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-dthqm, pid:6487, start at 2024-04-26 19:18:56.241902945 +0800 CST m=+5.772755438 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:20:56.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:18:56.244 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:08:56.244 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348983298457605, "checkpoint_time": "2024-04-26 19:18:57.900", "error": null }' + echo '{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348983298457605, "checkpoint_time": "2024-04-26 19:18:57.900", "error": null }' { "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348983298457605, "checkpoint_time": "2024-04-26 19:18:57.900", "error": null } ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348983298457605, '"checkpoint_time":' '"2024-04-26' '19:18:57.900",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348983298457605, '"checkpoint_time":' '"2024-04-26' '19:18:57.900",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11280.out cli changefeed --changefeed-id custom-changefeed-name remove =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 2-th check, retry later Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449348983560601608 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-5285?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... coverage: 2.1% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.7699.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > check diff successfully wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78ae21aa-6730-4c57-affb-3ba0ff4a7c88 {"id":"78ae21aa-6730-4c57-affb-3ba0ff4a7c88","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20ea1ef6 78ae21aa-6730-4c57-affb-3ba0ff4a7c88 /tidb/cdc/default/default/upstream/7362133660669749389 {"id":7362133660669749389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78ae21aa-6730-4c57-affb-3ba0ff4a7c88 {"id":"78ae21aa-6730-4c57-affb-3ba0ff4a7c88","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20ea1ef6 78ae21aa-6730-4c57-affb-3ba0ff4a7c88 /tidb/cdc/default/default/upstream/7362133660669749389 {"id":7362133660669749389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78ae21aa-6730-4c57-affb-3ba0ff4a7c88 {"id":"78ae21aa-6730-4c57-affb-3ba0ff4a7c88","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a20ea1ef6 78ae21aa-6730-4c57-affb-3ba0ff4a7c88 /tidb/cdc/default/default/upstream/7362133660669749389 {"id":7362133660669749389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.7628.out cli changefeed create --start-ts=449348982858579975 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-17139?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + set +x wait process cdc.test exit for 2-th time... table test.finish_mark exists check diff successfully Create changefeed successfully! ID: d123d3ba-f449-47f0-984c-be66642ea8e5 Info: {"upstream_id":7362133660669749389,"namespace":"default","id":"d123d3ba-f449-47f0-984c-be66642ea8e5","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-17139?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:19:01.66767541+08:00","start_ts":449348982858579975,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348982858579975,"checkpoint_ts":449348982858579975,"checkpoint_time":"2024-04-26 19:18:56.222"} PASS wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 3-th time... table drop_tables.c not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + set +x + tso='449348984124997633 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348984124997633 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:19:02 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.77247726.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:19:02 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> table big_txn.usertable1 exists cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:19:02 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> + set +x [Fri Apr 26 19:19:03 CST 2024] <<<<<< START kafka consumer in ddl_sequence case >>>>>> table drop_tables.c not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11335.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-5285?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7362133576518555571,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-5285?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:19:04.75721161+08:00","start_ts":449348985067929605,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348985067929605,"checkpoint_ts":449348985067929605,"checkpoint_time":"2024-04-26 19:19:04.650"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:05 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8df842f2-8c01-4927-b47d-ee1998124861 {"id":"8df842f2-8c01-4927-b47d-ee1998124861","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a210d09f6 8df842f2-8c01-4927-b47d-ee1998124861 /tidb/cdc/default/default/upstream/7362133697066527355 {"id":7362133697066527355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8df842f2-8c01-4927-b47d-ee1998124861 {"id":"8df842f2-8c01-4927-b47d-ee1998124861","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a210d09f6 8df842f2-8c01-4927-b47d-ee1998124861 /tidb/cdc/default/default/upstream/7362133697066527355 {"id":7362133697066527355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8df842f2-8c01-4927-b47d-ee1998124861 {"id":"8df842f2-8c01-4927-b47d-ee1998124861","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a210d09f6 8df842f2-8c01-4927-b47d-ee1998124861 /tidb/cdc/default/default/upstream/7362133697066527355 {"id":7362133697066527355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.7772.out cli changefeed create --start-ts=449348984124997633 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-source-test-21945?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 1dc1641b-abaa-4146-9e52-d7703af73d4d Info: {"upstream_id":7362133697066527355,"namespace":"default","id":"1dc1641b-abaa-4146-9e52-d7703af73d4d","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-source-test-21945?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:19:05.988519764+08:00","start_ts":449348984124997633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348984124997633,"checkpoint_ts":449348984124997633,"checkpoint_time":"2024-04-26 19:19:01.053"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table ddl_sequence.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 3-th check, retry later + set +x [Fri Apr 26 19:19:07 CST 2024] <<<<<< START kafka consumer in multi_source case >>>>>> go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 table ddl_sequence.finish_mark not exists for 2-th check, retry later Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c exists check diff successfully go: downloading golang.org/x/text v0.14.0 wait process cdc.test exit for 1-th time... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348986194886658, "checkpoint_time": "2024-04-26 19:19:08.949", "error": null }' + echo '{ "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348986194886658, "checkpoint_time": "2024-04-26 19:19:08.949", "error": null }' { "upstream_id": 7362133576518555571, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449348986194886658, "checkpoint_time": "2024-04-26 19:19:08.949", "error": null } ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348986194886658, '"checkpoint_time":' '"2024-04-26' '19:19:08.949",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362133576518555571, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449348986194886658, '"checkpoint_time":' '"2024-04-26' '19:19:08.949",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table ddl_sequence.finish_mark not exists for 3-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11434.out cli changefeed create --start-ts=449348976901881861 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-29442?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:19:10 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> [WARN] --tz is deprecated in changefeed settings. go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute v1.24.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 Create changefeed successfully! ID: 47e2cb70-bd61-426c-ada6-ad870cf40aff Info: {"upstream_id":7362133576518555571,"namespace":"default","id":"47e2cb70-bd61-426c-ada6-ad870cf40aff","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-29442?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-04-26T19:19:11.046653786+08:00","start_ts":449348976901881861,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348976901881861,"checkpoint_ts":449348976901881861,"checkpoint_time":"2024-04-26 19:18:33.499"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading go.opentelemetry.io/otel/metric v1.22.0 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11476.out cli unsafe delete-service-gc-safepoint table ddl_sequence.finish_mark not exists for 4-th check, retry later go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:03.570 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/04/26 19:19:03.570 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/04/26 19:19:03.964 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa5cbbce3_75ff_4f4a_9d5b_358f6f3cc198"] [2024/04/26 19:19:04.252 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8030eab0_a35c_4a46_bc91_004ed3ddfd47"] [2024/04/26 19:19:04.713 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9c454e25_6772_4ed1_87ea_47643e454592"] [2024/04/26 19:19:04.907 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8f1d16d6_8900_44a3_ba49_a9c6cfe3673b"] [2024/04/26 19:19:05.394 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9cf85684_066f_4379_8bf8_c7c968a10e12"] [2024/04/26 19:19:05.877 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs574d4624_e78a_4b83_93f0_05812dd2a942"] [2024/04/26 19:19:06.141 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/04/26 19:19:06.261 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:06.368 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:06.382 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs51f1c7b9_a4ae_49c4_91b3_59167b641639"] [2024/04/26 19:19:06.476 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse7cb23ee_0da6_45e7_b3f4_01521d81e529"] [2024/04/26 19:19:06.711 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:07.087 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:07.475 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:07.611 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:07.776 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/04/26 19:19:07.932 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:07.944 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:08.605 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:08.639 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:08.846 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/04/26 19:19:09.116 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:09.410 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:09.431 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:09.474 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/04/26 19:19:09.702 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/04/26 19:19:09.751 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:09.920 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:10.069 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:10.075 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:10.149 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:10.380 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:10.623 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:11.213 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:11.364 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:11.596 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:11.620 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:12.227 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:12.338 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:12.495 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/04/26 19:19:12.514 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/04/26 19:19:12.588 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/04/26 19:19:12.804 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:13.005 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:19:13.093 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/04/26 19:19:13.105 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:13.280 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:13.613 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:13.633 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:19:13.671 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:13.675 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:13.794 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:19:14.021 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11500.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:19:14.497 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:19:14.531 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:19:14.666 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] table ddl_sequence.finish_mark not exists for 5-th check, retry later [2024/04/26 19:19:14.694 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/04/26 19:19:14.924 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6885e0180012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:6210, start at 2024-04-26 19:19:12.156837261 +0800 CST m=+5.667410587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:12.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:12.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:12.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6885e0180012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:6210, start at 2024-04-26 19:19:12.156837261 +0800 CST m=+5.667410587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:12.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:12.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:12.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6885e2600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-8fmzz, pid:6289, start at 2024-04-26 19:19:12.325427922 +0800 CST m=+5.738679762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:12.333 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:12.331 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:12.331 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:19:13 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh: line 1: 12671 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/new_changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:19:15.106 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:15.472 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:19:15.580 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + set +x [2024/04/26 19:19:15.796 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:19:15.871 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/04/26 19:19:15.904 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:19:15.931 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table ddl_sequence.finish_mark exists check diff successfully [2024/04/26 19:19:16.345 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/04/26 19:19:16.502 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7526.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 wait process cdc.test exit for 1-th time... [2024/04/26 19:19:16.898 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:19:16.940 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/04/26 19:19:16.978 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:19:17.093 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:19:17.094 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:19:17.121 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd wait process cdc.test exit for 2-th time... [2024/04/26 19:19:17.565 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:17.820 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:19:17.822 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:19:18.387 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:19:18.398 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:19:18.580 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] wait process cdc.test exit for 3-th time... + set +x + tso='449348988340535301 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348988340535301 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:19:18 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.75497551.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11597.out cli unsafe resolve-lock --region=66 [2024/04/26 19:19:19.072 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:19:19 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:19:19.201 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/04/26 19:19:19.220 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:19:19.293 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:19:19.300 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/04/26 19:19:19.393 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:19:19.603 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:19.883 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... [2024/04/26 19:19:19.967 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:19:19.983 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:19:20.103 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:19:20.198 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:19:20.279 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11620.out cli unsafe resolve-lock --region=66 --ts=449348988017049602 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:19:20.896 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:19:20.911 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:19:21.361 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:19:21.794 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:19:21.802 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/04/26 19:19:21.858 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/351904c8-0639-434c-8de7-9fb382812532 {"id":"351904c8-0639-434c-8de7-9fb382812532","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a214ff6f6 351904c8-0639-434c-8de7-9fb382812532 /tidb/cdc/default/default/upstream/7362133767453381190 {"id":7362133767453381190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/351904c8-0639-434c-8de7-9fb382812532 {"id":"351904c8-0639-434c-8de7-9fb382812532","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a214ff6f6 351904c8-0639-434c-8de7-9fb382812532 /tidb/cdc/default/default/upstream/7362133767453381190 {"id":7362133767453381190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/351904c8-0639-434c-8de7-9fb382812532 {"id":"351904c8-0639-434c-8de7-9fb382812532","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a214ff6f6 351904c8-0639-434c-8de7-9fb382812532 /tidb/cdc/default/default/upstream/7362133767453381190 {"id":7362133767453381190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.7601.out cli changefeed create --start-ts=449348988340535301 '--sink-uri=kafka://127.0.0.1:9092/ticdc-common-1-test-14622?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/04/26 19:19:21.955 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:19:22.025 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Create changefeed successfully! ID: 521c910b-a333-47cf-97f3-3ef58f743510 Info: {"upstream_id":7362133767453381190,"namespace":"default","id":"521c910b-a333-47cf-97f3-3ef58f743510","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-14622?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:19:22.075051596+08:00","start_ts":449348988340535301,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348988340535301,"checkpoint_ts":449348988340535301,"checkpoint_time":"2024-04-26 19:19:17.134"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 156k 4528 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" [2024/04/26 19:19:22.624 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:19:22.645 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/04/26 19:19:22.667 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:19:22.676 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:19:20.678 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 30"] Verifying downstream PD is started... [2024/04/26 19:19:23.078 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/04/26 19:19:23.175 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:19:23.389 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:19:23 CST 2024] <<<<<< START kafka consumer in common_1 case >>>>>> [2024/04/26 19:19:23.575 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:19:23.702 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:19:23.787 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:19:23.953 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/04/26 19:19:23.995 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/04/26 19:19:24.029 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:19:24.037 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:24.393 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:19:24.507 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:19:24.790 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 176 100 176 0 0 211k 0 --:--:-- --:--:-- --:--:-- 171k { "version": "v7.5.1-57-g980f60a5b", "git_hash": "980f60a5b0a82befa601d05e3558ec8795625b13", "id": "bef262da-b1bc-45bb-8579-cdfaa42da0c7", "pid": 10797, "is_owner": true }wait process cdc.test exit for 1-th time... [2024/04/26 19:19:25.899 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:19:26.020 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:19:26.127 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:19:26 CST 2024] <<<<<< run test case cli success! >>>>>> [2024/04/26 19:19:26.849 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:19:26.899 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:19:27.014 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/04/26 19:19:27.066 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:19:27.393 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/04/26 19:19:27.475 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:19:27.792 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/04/26 19:19:27.898 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:28.381 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:19:28.480 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:19:28.524 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] skip some SQLs in tidb v4.0.x table common_1.v1 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:29.105 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:19:29.121 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table common_1.recover_and_insert not exists for 1-th check, retry later [2024/04/26 19:19:29.431 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:19:29.525 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:19:29.582 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:19:29 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 10 [Fri Apr 26 19:19:29 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 20 [Fri Apr 26 19:19:29 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later [2024/04/26 19:19:30.376 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/04/26 19:19:30.468 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6886ee480017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:13584, start at 2024-04-26 19:19:29.468175948 +0800 CST m=+5.240710804 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:29.475 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:29.476 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:29.476 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6886ee480017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:13584, start at 2024-04-26 19:19:29.468175948 +0800 CST m=+5.240710804 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:29.475 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:29.476 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:29.476 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6886efb00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:13658, start at 2024-04-26 19:19:29.561341388 +0800 CST m=+5.228528370 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:29.567 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:29.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:29.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:19:30.783 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.recover_and_insert not exists for 2-th check, retry later [2024/04/26 19:19:31.110 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:19:31.425 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:19:31.592 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish not exists for 2-th check, retry later [2024/04/26 19:19:31.879 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/04/26 19:19:31.904 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... [2024/04/26 19:19:32.081 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:19:32.298 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:19:32.490 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [Fri Apr 26 19:19:32 CST 2024] <<<<<< START cdc server in kafka_column_selector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.1500615008.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.recover_and_insert not exists for 3-th check, retry later [2024/04/26 19:19:33.092 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:19:33.210 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:19:33.212 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68871e340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:6292, start at 2024-04-26 19:19:32.523284705 +0800 CST m=+25.206649166 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:32.535 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:32.493 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:32.493 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68871e340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:6292, start at 2024-04-26 19:19:32.523284705 +0800 CST m=+25.206649166 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:32.535 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:32.493 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:32.493 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688605b00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-gkqgk, pid:6367, start at 2024-04-26 19:19:14.57706783 +0800 CST m=+7.154417880 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:14.585 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:14.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:14.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:19:33.906 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:19:33.995 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:19:33.995 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:19:34.005 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/04/26 19:19:34.006 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:19:34.090 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] table test.finish exists check diff successfully [2024/04/26 19:19:34.288 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process cdc.test exit for 1-th time... [2024/04/26 19:19:34.695 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:34.896 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:19:35 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later [2024/04/26 19:19:35.669 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:19:35.985 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:19:36.095 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/812214ce-2e52-4fe0-a261-ce5b29139a92 {"id":"812214ce-2e52-4fe0-a261-ce5b29139a92","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21931ff1 812214ce-2e52-4fe0-a261-ce5b29139a92 /tidb/cdc/default/default/upstream/7362133839594367678 {"id":7362133839594367678,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/812214ce-2e52-4fe0-a261-ce5b29139a92 {"id":"812214ce-2e52-4fe0-a261-ce5b29139a92","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21931ff1 812214ce-2e52-4fe0-a261-ce5b29139a92 /tidb/cdc/default/default/upstream/7362133839594367678 {"id":7362133839594367678,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/812214ce-2e52-4fe0-a261-ce5b29139a92 {"id":"812214ce-2e52-4fe0-a261-ce5b29139a92","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21931ff1 812214ce-2e52-4fe0-a261-ce5b29139a92 /tidb/cdc/default/default/upstream/7362133839594367678 {"id":7362133839594367678,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.cli.15057.out cli changefeed create --start-ts=449348992414777345 '--sink-uri=kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json&partition-num=1&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/conf/changefeed.toml Create changefeed successfully! ID: test Info: {"upstream_id":7362133839594367678,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json\u0026partition-num=1\u0026enable-tidb-extension=true","create_time":"2024-04-26T19:19:36.207978288+08:00","start_ts":449348992414777345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":[{"matcher":["test.t1"],"columns":["a","b"]},{"matcher":["test.*"],"columns":["*","!b"]},{"matcher":["test1.t1"],"columns":["column*","!column1"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348992414777345,"checkpoint_ts":449348992414777345,"checkpoint_time":"2024-04-26 19:19:32.676"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Fri Apr 26 19:19:35 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1996319965.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:19:36.429 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:19:36.500 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/04/26 19:19:36.579 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/04/26 19:19:36.691 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68874fac0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:5919, start at 2024-04-26 19:19:35.670902724 +0800 CST m=+9.120922768 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:35.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:35.659 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:35.659 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68874fac0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:5919, start at 2024-04-26 19:19:35.670902724 +0800 CST m=+9.120922768 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:35.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:35.659 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:35.659 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68874d240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-lxpkt, pid:5954, start at 2024-04-26 19:19:35.542295279 +0800 CST m=+8.790096120 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:35.558 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:35.546 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:35.546 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/04/26 19:19:36.818 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/04/26 19:19:37.017 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:19:37.274 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore table common_1.finish_mark not exists for 2-th check, retry later + set +x [2024/04/26 19:19:37.700 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:19:37.890 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:38.068 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:19:38.390 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... [2024/04/26 19:19:38.516 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:19:38.591 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:19:38.676 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:19:38.775 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:19:38.985 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.7141.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:19:39.369 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table common_1.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:38 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/330a07fe-ff8d-48e6-bd11-a17b6a0ceff2 {"id":"330a07fe-ff8d-48e6-bd11-a17b6a0ceff2","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a214cac11 330a07fe-ff8d-48e6-bd11-a17b6a0ceff2 /tidb/cdc/default/default/upstream/7362133764107656956 {"id":7362133764107656956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/330a07fe-ff8d-48e6-bd11-a17b6a0ceff2 {"id":"330a07fe-ff8d-48e6-bd11-a17b6a0ceff2","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a214cac11 330a07fe-ff8d-48e6-bd11-a17b6a0ceff2 /tidb/cdc/default/default/upstream/7362133764107656956 {"id":7362133764107656956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/330a07fe-ff8d-48e6-bd11-a17b6a0ceff2 {"id":"330a07fe-ff8d-48e6-bd11-a17b6a0ceff2","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a214cac11 330a07fe-ff8d-48e6-bd11-a17b6a0ceff2 /tidb/cdc/default/default/upstream/7362133764107656956 {"id":7362133764107656956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.20331.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-rocks-test-16633?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: e9b1d696-31f8-4deb-9e65-df4d3648b85b Info: {"upstream_id":7362133764107656956,"namespace":"default","id":"e9b1d696-31f8-4deb-9e65-df4d3648b85b","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-rocks-test-16633?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:19:39.420546999+08:00","start_ts":449348994136276996,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348994136276996,"checkpoint_ts":449348994136276996,"checkpoint_time":"2024-04-26 19:19:39.243"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:39.912 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:19:40.120 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:19:40.388 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/04/26 19:19:40.593 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + set +x [Fri Apr 26 19:19:40 CST 2024] <<<<<< START kafka consumer in multi_rocks case >>>>>> start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... ***************** properties ***************** "mysql.db"="multi_rocks" "recordcount"="1000" "readallfields"="true" "mysql.host"="127.0.0.1" "threadcount"="2" "updateproportion"="0" "insertproportion"="0" "dotransactions"="false" "mysql.port"="4000" "requestdistribution"="uniform" "readproportion"="0" "mysql.user"="root" "workload"="core" "operationcount"="0" "scanproportion"="0" "table"="a1" ********************************************** Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:19:41.093 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table common_1.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:41.402 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] Run finished, takes 554.168335ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1845.4, Avg(us): 1075, Min(us): 783, Max(us): 12175, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "operationcount"="0" "dotransactions"="false" "mysql.port"="4000" "mysql.db"="multi_rocks" "workload"="core" "mysql.user"="root" "scanproportion"="0" "insertproportion"="0" "readproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "recordcount"="1000" "mysql.host"="127.0.0.1" "readallfields"="true" "updateproportion"="0" "table"="a2" ********************************************** ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting build checksum checker... go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/klauspost/compress v1.17.7 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading github.com/gin-gonic/gin v1.8.1 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading github.com/tinylib/msgp v1.1.6 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 [2024/04/26 19:19:41.721 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:19:41.726 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/04/26 19:19:41.889 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] + set +x + tso='449348994284650497 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348994284650497 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "scanproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "workload"="core" "mysql.user"="root" "mysql.db"="multi_capture_1" "dotransactions"="false" "readallfields"="true" "requestdistribution"="uniform" "readproportion"="0" "recordcount"="10" "threadcount"="2" "mysql.port"="4000" "updateproportion"="0" ********************************************** Run finished, takes 16.53897ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1112.1, Avg(us): 3168, Min(us): 1783, Max(us): 7443, 95th(us): 8000, 99th(us): 8000 wait process cdc.test exit for 1-th time... go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/time v0.5.0 ***************** properties ***************** "recordcount"="10" "threadcount"="2" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.db"="multi_capture_2" "readproportion"="0" "readallfields"="true" "updateproportion"="0" "mysql.port"="4000" "scanproportion"="0" "workload"="core" "requestdistribution"="uniform" "operationcount"="0" "insertproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 17.368948ms INSERT - Takes(s): 0.0, Count: 9, OPS: 927.4, Avg(us): 2868, Min(us): 2010, Max(us): 7525, 95th(us): 8000, 99th(us): 8000 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 Run finished, takes 624.086838ms INSERT - Takes(s): 0.6, Count: 1000, OPS: 1657.9, Avg(us): 1210, Min(us): 825, Max(us): 20997, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "recordcount"="10" "mysql.port"="4000" "scanproportion"="0" "insertproportion"="0" "readallfields"="true" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_3" "workload"="core" "updateproportion"="0" "operationcount"="0" "mysql.user"="root" "threadcount"="2" "dotransactions"="false" "requestdistribution"="uniform" ********************************************** Run finished, takes 16.13201ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1005.6, Avg(us): 3092, Min(us): 1772, Max(us): 6054, 95th(us): 7000, 99th(us): 7000 [2024/04/26 19:19:42.390 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:19:42.406 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] wait process cdc.test exit for 2-th time... ***************** properties ***************** "threadcount"="2" go: downloading github.com/philhofer/fwd v1.1.1 "recordcount"="1000" "requestdistribution"="uniform" go: downloading github.com/spf13/pflag v1.0.5 "table"="a3" "workload"="core" go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc "mysql.host"="127.0.0.1" "updateproportion"="0" go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 "scanproportion"="0" go: downloading github.com/stretchr/objx v0.5.2 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/gin-contrib/sse v0.1.0 "mysql.db"="multi_rocks" "dotransactions"="false" go: downloading github.com/mattn/go-isatty v0.0.20 "mysql.user"="root" "readallfields"="true" go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.uber.org/multierr v1.11.0 "insertproportion"="0" "readproportion"="0" go: downloading github.com/cilium/ebpf v0.4.0 "mysql.port"="4000" "operationcount"="0" ********************************************** ***************** properties ***************** "mysql.user"="root" "readallfields"="true" "readproportion"="0" "scanproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "mysql.port"="4000" "insertproportion"="0" "threadcount"="2" "recordcount"="10" "requestdistribution"="uniform" "operationcount"="0" "mysql.db"="multi_capture_4" "updateproportion"="0" "workload"="core" ********************************************** Run finished, takes 17.442785ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1043.4, Avg(us): 3376, Min(us): 1877, Max(us): 7763, 95th(us): 8000, 99th(us): 8000 [Fri Apr 26 19:19:42 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.72457247.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:19:42.681 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/go-playground/validator/v10 v10.11.1 go: downloading github.com/pelletier/go-toml/v2 v2.0.5 go: downloading github.com/ugorji/go/codec v1.2.7 go: downloading google.golang.org/protobuf v1.33.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:19:43.012 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] go: downloading gopkg.in/yaml.v2 v2.4.0 cdc.test: no process found wait process cdc.test exit for 3-th time... Run finished, takes 610.112417ms Verifying downstream PD is started... INSERT - Takes(s): 0.6, Count: 1000, OPS: 1691.4, Avg(us): 1185, Min(us): 797, Max(us): 18757, 95th(us): 2000, 99th(us): 2000 process cdc.test already exit go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 [Fri Apr 26 19:19:42 CST 2024] <<<<<< run test case common_1 success! >>>>>> ***************** properties ***************** "dotransactions"="false" "readallfields"="true" go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 "mysql.host"="127.0.0.1" go: downloading github.com/aws/smithy-go v1.13.5 "mysql.port"="4000" "operationcount"="0" "mysql.user"="root" "scanproportion"="0" "readproportion"="0" "workload"="core" "insertproportion"="0" "recordcount"="1000" "table"="a4" "threadcount"="2" "mysql.db"="multi_rocks" "updateproportion"="0" "requestdistribution"="uniform" ********************************************** [2024/04/26 19:19:43.123 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:19:43.190 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:19:43.289 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:43.383 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:19:43.396 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:19:43.480 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:19:43.486 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/golang/protobuf v1.5.4 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/oauth2 v0.18.0 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading google.golang.org/api v0.162.0 Run finished, takes 827.86889ms INSERT - Takes(s): 0.8, Count: 1000, OPS: 1236.9, Avg(us): 1618, Min(us): 786, Max(us): 80335, 95th(us): 2000, 99th(us): 7000 ***************** properties ***************** "insertproportion"="0" "threadcount"="2" "mysql.user"="root" "table"="a5" "dotransactions"="false" "readallfields"="true" "operationcount"="0" "mysql.port"="4000" "mysql.db"="multi_rocks" "workload"="core" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "recordcount"="1000" "readproportion"="0" "updateproportion"="0" "scanproportion"="0" ********************************************** go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/go-playground/universal-translator v0.18.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/leodido/go-urn v1.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/go-playground/locales v0.14.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da Starting Upstream TiDB... Run finished, takes 870.293296ms INSERT - Takes(s): 0.9, Count: 1000, OPS: 1172.1, Avg(us): 1706, Min(us): 832, Max(us): 47027, 95th(us): 2000, 99th(us): 28000 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading github.com/go-logr/stdr v1.2.2 Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6887d54c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:6293, start at 2024-04-26 19:19:44.259097943 +0800 CST m=+5.111566909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:44.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:44.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:44.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6887d54c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:6293, start at 2024-04-26 19:19:44.259097943 +0800 CST m=+5.111566909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:44.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:44.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:44.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6887d8540006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:6359, start at 2024-04-26 19:19:44.412047839 +0800 CST m=+5.167605126 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:44.418 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:44.405 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:44.405 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:45.070 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:19:45.517 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table multi_rocks.finish_mark not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:45.601 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:45 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:19:45 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.72917293.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:19:45.908 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/04/26 19:19:46.074 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:19:46.115 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:46.180 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:19:46.981 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:19:47.119 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:19:47.266 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:19:47 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.75817583.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info [2024/04/26 19:19:47.419 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:19:47.572 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=kafka + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run kafka + '[' kafka '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Fri Apr 26 19:19:46 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Fri Apr 26 19:19:46 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688803640005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:5540, start at 2024-04-26 19:19:47.168360653 +0800 CST m=+9.691668484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:47.237 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:47.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:47.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688803640005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:5540, start at 2024-04-26 19:19:47.168360653 +0800 CST m=+9.691668484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:47.237 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:47.261 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:47.261 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68880af0000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:5574, start at 2024-04-26 19:19:47.673107959 +0800 CST m=+10.014009611 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:47.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:47.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:47.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/ardielle/ardielle-go v1.5.2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:48.619 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/04/26 19:19:48.619 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:19:48.677 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dolthub/maphash v0.1.0 + (( i++ )) + (( i <= 50 )) go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef [2024/04/26 19:19:49.015 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] go: downloading github.com/tklauser/go-sysconf v0.3.12 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 > < HTTP/1.1 200 OK go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 < Date: Fri, 26 Apr 2024 11:19:48 GMT < Content-Length: 867 go: downloading github.com/tklauser/numcpus v0.6.1 < Content-Type: text/plain; charset=utf-8 < { [data not shown] go: downloading github.com/kylelemons/godebug v1.1.0 * Connection #0 to host 127.0.0.1 left intact + res=' go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c *** processors info ***: *** etcd info ***: go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/e6ed299c-282d-4066-b3ca-064321f533f5 {"id":"e6ed299c-282d-4066-b3ca-064321f533f5","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e978 e6ed299c-282d-4066-b3ca-064321f533f5 /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/e6ed299c-282d-4066-b3ca-064321f533f5 {"id":"e6ed299c-282d-4066-b3ca-064321f533f5","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e978 e6ed299c-282d-4066-b3ca-064321f533f5 /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/e6ed299c-282d-4066-b3ca-064321f533f5 {"id":"e6ed299c-282d-4066-b3ca-064321f533f5","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e978 e6ed299c-282d-4066-b3ca-064321f533f5 /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:19:48 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.73337335.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: kafka... <<================= table multi_rocks.finish_mark not exists for 2-th check, retry later [2024/04/26 19:19:49.287 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] go: downloading github.com/getsentry/sentry-go v0.27.0 [2024/04/26 19:19:49.572 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] go: downloading github.com/cockroachdb/redact v1.1.5 [Fri Apr 26 19:19:49 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b [2024/04/26 19:19:49.652 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] go: downloading github.com/kr/pretty v0.3.1 [2024/04/26 19:19:49.652 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:49.872 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/04/26 19:19:50.399 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:19:50.819 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [Fri Apr 26 19:19:50 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.67476749.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:50 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ebae5ed5-5b65-42e1-8056-1207fe134701 {"id":"ebae5ed5-5b65-42e1-8056-1207fe134701","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d2f4 ebae5ed5-5b65-42e1-8056-1207fe134701 /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ebae5ed5-5b65-42e1-8056-1207fe134701 {"id":"ebae5ed5-5b65-42e1-8056-1207fe134701","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d2f4 ebae5ed5-5b65-42e1-8056-1207fe134701 /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ebae5ed5-5b65-42e1-8056-1207fe134701 {"id":"ebae5ed5-5b65-42e1-8056-1207fe134701","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d2f4 ebae5ed5-5b65-42e1-8056-1207fe134701 /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.7636.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7362133901770366895,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:19:51.065365409+08:00","start_ts":449348997195235329,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348997195235329,"checkpoint_ts":449348997195235329,"checkpoint_time":"2024-04-26 19:19:50.912"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table multi_rocks.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:51 GMT < Content-Length: 1144 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/c8e5f006-f482-44a6-b25f-d8e4a218524c {"id":"c8e5f006-f482-44a6-b25f-d8e4a218524c","address":"127.0.0.1:8303","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/e6ed299c-282d-4066-b3ca-064321f533f5 {"id":"e6ed299c-282d-4066-b3ca-064321f533f5","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e978 e6ed299c-282d-4066-b3ca-064321f533f5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e987 c8e5f006-f482-44a6-b25f-d8e4a218524c /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/c8e5f006-f482-44a6-b25f-d8e4a218524c {"id":"c8e5f006-f482-44a6-b25f-d8e4a218524c","address":"127.0.0.1:8303","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/e6ed299c-282d-4066-b3ca-064321f533f5 {"id":"e6ed299c-282d-4066-b3ca-064321f533f5","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e978 e6ed299c-282d-4066-b3ca-064321f533f5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e987 c8e5f006-f482-44a6-b25f-d8e4a218524c /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d13be59-4480-4621-8b3c-84e1a7a97afc {"id":"1d13be59-4480-4621-8b3c-84e1a7a97afc","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/c8e5f006-f482-44a6-b25f-d8e4a218524c {"id":"c8e5f006-f482-44a6-b25f-d8e4a218524c","address":"127.0.0.1:8303","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/e6ed299c-282d-4066-b3ca-064321f533f5 {"id":"e6ed299c-282d-4066-b3ca-064321f533f5","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e94c 1d13be59-4480-4621-8b3c-84e1a7a97afc /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e978 e6ed299c-282d-4066-b3ca-064321f533f5 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2197e987 c8e5f006-f482-44a6-b25f-d8e4a218524c /tidb/cdc/default/default/upstream/7362133845305114513 {"id":7362133845305114513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.7368.out cli changefeed create --start-ts=449348994284650497 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-capture-test-7203?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8301 Create changefeed successfully! ID: 270ea772-8977-41c6-b798-551c207a2035 Info: {"upstream_id":7362133845305114513,"namespace":"default","id":"270ea772-8977-41c6-b798-551c207a2035","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-capture-test-7203?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:19:52.516107683+08:00","start_ts":449348994284650497,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348994284650497,"checkpoint_ts":449348994284650497,"checkpoint_time":"2024-04-26 19:19:39.809"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:19:52 CST 2024] <<<<<< START kafka consumer in ddl_only_block_related_table case >>>>>> [2024/04/26 19:19:52.284 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/04/26 19:19:52.354 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/04/26 19:19:52.450 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] coverage: 2.5% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:19:52 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> [2024/04/26 19:19:52.786 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf38e1e31_1e82_4329_aabf_49d04edf889f"] [2024/04/26 19:19:53.304 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs513320a2_52f9_4868_93c9_a87eb0770fff"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:19:53 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/284f72e5-f583-444f-8c5f-95273dd0630e {"id":"284f72e5-f583-444f-8c5f-95273dd0630e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6adf7 284f72e5-f583-444f-8c5f-95273dd0630e /tidb/cdc/default/default/upstream/7362133898101840730 {"id":7362133898101840730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/284f72e5-f583-444f-8c5f-95273dd0630e {"id":"284f72e5-f583-444f-8c5f-95273dd0630e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6adf7 284f72e5-f583-444f-8c5f-95273dd0630e /tidb/cdc/default/default/upstream/7362133898101840730 {"id":7362133898101840730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/284f72e5-f583-444f-8c5f-95273dd0630e {"id":"284f72e5-f583-444f-8c5f-95273dd0630e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6adf7 284f72e5-f583-444f-8c5f-95273dd0630e /tidb/cdc/default/default/upstream/7362133898101840730 {"id":7362133898101840730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:19:53 CST 2024] <<<<<< START kafka consumer in processor_stop_delay case >>>>>> table multi_rocks.finish_mark not exists for 4-th check, retry later [2024/04/26 19:19:53.974 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs511d24e2_19e9_4e5b_9112_94ca45356836"] [2024/04/26 19:19:54.116 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs30c6aac3_6bc1_404e_92da_b4a59cc22a63"] [2024/04/26 19:19:54.208 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3bc688f6_c9e3_41e6_8063_b2c34a077ec5"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68884b7c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:12535, start at 2024-04-26 19:19:51.798300854 +0800 CST m=+5.575807784 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:51.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:51.775 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:51.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68884b7c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:12535, start at 2024-04-26 19:19:51.798300854 +0800 CST m=+5.575807784 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:51.805 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:51.775 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:51.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68884c200013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:12592, start at 2024-04-26 19:19:51.838198164 +0800 CST m=+5.534107987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:51.844 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:51.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:51.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_stop_delay.t not exists for 1-th check, retry later + set +x [Fri Apr 26 19:19:54 CST 2024] <<<<<< START kafka consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd [2024/04/26 19:19:54.864 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/04/26 19:19:55.209 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs54f057ae_2c9a_42a0_b938_add64b1e7d7a"] table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [2024/04/26 19:19:55.377 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs551572ff_cec4_4408_b1ec_d6e47a12e391"] [Pipeline] // cache [Pipeline] } [Pipeline] // dir VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688866e8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:8423, start at 2024-04-26 19:19:53.553347521 +0800 CST m=+8.469375534 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:53.571 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:53.579 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:53.579 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688866e8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:8423, start at 2024-04-26 19:19:53.553347521 +0800 CST m=+8.469375534 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:53.571 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:53.579 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:53.579 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688868640002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-wqb6h, pid:8458, start at 2024-04-26 19:19:53.627524933 +0800 CST m=+8.375188842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:21:53.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:19:53.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:09:53.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] } [2024/04/26 19:19:55.695 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } [Pipeline] // stage [2024/04/26 19:19:55.821 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc24f167d_dd22_4af9_bbd8_fae51474dfcf"] [2024/04/26 19:19:55.835 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [Pipeline] } table multi_rocks.finish_mark exists check diff successfully [Pipeline] // container [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:19:55 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } wait process cdc.test exit for 1-th time... table multi_capture_1.usertable not exists for 2-th check, retry later [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } [2024/04/26 19:19:56.592 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:56.622 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/04/26 19:19:56.695 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/04/26 19:19:56.875 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.13789.out cli tso query --pd=http://127.0.0.1:2379 table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:19:57 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> [2024/04/26 19:19:57.204 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:57.285 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:57.409 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table processor_stop_delay.t exists check diff failed 1-th time, retry later [2024/04/26 19:19:57.785 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:57.826 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:57.846 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + set +x + tso='449348998758400001 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348998758400001 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:19:58 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + GO_FAILPOINTS= + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1381813820.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.9756.out cli tso query --pd=http://127.0.0.1:2379 table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [2024/04/26 19:19:58.881 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/04/26 19:19:58.961 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [2024/04/26 19:19:59.270 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [2024/04/26 19:19:59.342 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:19:59.351 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:19:59.391 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:19:59.500 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later [Pipeline] // stage [Pipeline] } [2024/04/26 19:19:59.725 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [Pipeline] // container [Pipeline] } [2024/04/26 19:19:59.853 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [Pipeline] // withEnv [Pipeline] } + set +x + tso='449348999126450177 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449348999126450177 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:20:00 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.97769778.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // node check diff successfully [Pipeline] } [2024/04/26 19:20:00.377 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/04/26 19:20:01.022 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:01.193 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table ddl_only_block_related_table.finish_mark exists [2024/04/26 19:20:00.184 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`many_cols`] [where="((TRUE) AND TRUE)"] ["source checksum"=2933305990] ["target checksum"=3362591174] ["get source checksum cost"=1.676808ms] ["get target checksum cost"=1.228683ms] [2024/04/26 19:20:00.186 +08:00] [WARN] [diff.go:893] ["find different row"] [column=val] [row1="{ val: 2009, id: 1, }"] [row2="{ val: 1958, id: 1, }"] [2024/04/26 19:20:00.275 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`many_cols`] [where="((TRUE) AND TRUE)"] [cost=90.91787ms] [2024/04/26 19:20:00.277 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [2024/04/26 19:20:01.387 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:01.406 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:01.406 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:01.417 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:20:01 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bbfe032e-75d1-460a-ba14-0888fc33f431 {"id":"bbfe032e-75d1-460a-ba14-0888fc33f431","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21e90df5 bbfe032e-75d1-460a-ba14-0888fc33f431 /tidb/cdc/default/default/upstream/7362133932934272124 {"id":7362133932934272124,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bbfe032e-75d1-460a-ba14-0888fc33f431 {"id":"bbfe032e-75d1-460a-ba14-0888fc33f431","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21e90df5 bbfe032e-75d1-460a-ba14-0888fc33f431 /tidb/cdc/default/default/upstream/7362133932934272124 {"id":7362133932934272124,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bbfe032e-75d1-460a-ba14-0888fc33f431 {"id":"bbfe032e-75d1-460a-ba14-0888fc33f431","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21e90df5 bbfe032e-75d1-460a-ba14-0888fc33f431 /tidb/cdc/default/default/upstream/7362133932934272124 {"id":7362133932934272124,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.13880.out cli changefeed create --start-ts=449348998758400001 '--sink-uri=kafka://127.0.0.1:9092/ticdc-savepoint-test-25192?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' wait process 7586 exit for 1-th time... Create changefeed successfully! ID: 7caaee2d-579f-47b2-bdef-79914e6ebcfe Info: {"upstream_id":7362133932934272124,"namespace":"default","id":"7caaee2d-579f-47b2-bdef-79914e6ebcfe","sink_uri":"kafka://127.0.0.1:9092/ticdc-savepoint-test-25192?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:20:01.787819429+08:00","start_ts":449348998758400001,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348998758400001,"checkpoint_ts":449348998758400001,"checkpoint_time":"2024-04-26 19:19:56.875"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:20:01.878 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff failed 1-th time, retry later wait process 7586 exit for 2-th time... [2024/04/26 19:20:02.208 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/04/26 19:20:02.708 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7586) - No such process wait process 7586 exit for 3-th time... process 7586 already exit [Fri Apr 26 19:20:02 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.77477749.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_capture_4.usertable exists check diff failed 1-th time, retry later [2024/04/26 19:20:02.970 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + set +x [Fri Apr 26 19:20:03 CST 2024] <<<<<< START kafka consumer in savepoint case >>>>>> [2024/04/26 19:20:03.144 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:03.184 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:20:03 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c6248ad-58a3-4606-8af9-922377a93453 {"id":"5c6248ad-58a3-4606-8af9-922377a93453","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21e07501 5c6248ad-58a3-4606-8af9-922377a93453 /tidb/cdc/default/default/upstream/7362133927505663295 {"id":7362133927505663295,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c6248ad-58a3-4606-8af9-922377a93453 {"id":"5c6248ad-58a3-4606-8af9-922377a93453","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21e07501 5c6248ad-58a3-4606-8af9-922377a93453 /tidb/cdc/default/default/upstream/7362133927505663295 {"id":7362133927505663295,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c6248ad-58a3-4606-8af9-922377a93453 {"id":"5c6248ad-58a3-4606-8af9-922377a93453","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21e07501 5c6248ad-58a3-4606-8af9-922377a93453 /tidb/cdc/default/default/upstream/7362133927505663295 {"id":7362133927505663295,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.9819.out cli changefeed create --start-ts=449348999126450177 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-13798?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/04/26 19:20:03.386 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:03.513 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Create changefeed successfully! ID: 3f2b0977-7cf6-4e30-ad5f-e6c6fde463b9 Info: {"upstream_id":7362133927505663295,"namespace":"default","id":"3f2b0977-7cf6-4e30-ad5f-e6c6fde463b9","sink_uri":"kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-13798?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:20:03.75118559+08:00","start_ts":449348999126450177,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449348999126450177,"checkpoint_ts":449348999126450177,"checkpoint_time":"2024-04-26 19:19:58.279"} PASS [2024/04/26 19:20:03.597 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:03.778 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:20:03.927 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:03.981 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] check diff failed 2-th time, retry later table savepoint.finish_mark not exists for 1-th check, retry later + set +x [Fri Apr 26 19:20:05 CST 2024] <<<<<< START kafka consumer in resourcecontrol case >>>>>> check diff successfully ***************** properties ***************** "mysql.port"="4000" "updateproportion"="0" "mysql.db"="multi_capture_1" "threadcount"="2" "operationcount"="0" "requestdistribution"="uniform" "dotransactions"="false" "recordcount"="20" "scanproportion"="0" "mysql.user"="root" "insertproportion"="0" "readallfields"="true" "readproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 66.019192ms INSERT - Takes(s): 0.1, Count: 20, OPS: 314.3, Avg(us): 5278, Min(us): 1509, Max(us): 10540, 95th(us): 11000, 99th(us): 11000 ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.port"="4000" "dotransactions"="false" "insertproportion"="0" "readallfields"="true" "updateproportion"="0" "scanproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "workload"="core" "threadcount"="2" "operationcount"="0" "mysql.user"="root" "mysql.db"="multi_capture_2" "recordcount"="20" ********************************************** Run finished, takes 50.599547ms INSERT - Takes(s): 0.0, Count: 20, OPS: 420.0, Avg(us): 4640, Min(us): 1427, Max(us): 9861, 95th(us): 9000, 99th(us): 10000 ***************** properties ***************** "updateproportion"="0" "readallfields"="true" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_3" "dotransactions"="false" "readproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "threadcount"="2" "recordcount"="20" "scanproportion"="0" "mysql.user"="root" "workload"="core" "mysql.port"="4000" ********************************************** Run finished, takes 43.721457ms INSERT - Takes(s): 0.0, Count: 20, OPS: 502.9, Avg(us): 3819, Min(us): 1296, Max(us): 6809, 95th(us): 7000, 99th(us): 7000 ***************** properties ***************** "mysql.port"="4000" "recordcount"="20" "threadcount"="2" "mysql.user"="root" "mysql.host"="127.0.0.1" "dotransactions"="false" "readallfields"="true" "updateproportion"="0" "requestdistribution"="uniform" "mysql.db"="multi_capture_4" "workload"="core" "insertproportion"="0" "readproportion"="0" "operationcount"="0" "scanproportion"="0" ********************************************** Run finished, takes 44.820888ms INSERT - Takes(s): 0.0, Count: 20, OPS: 548.8, Avg(us): 4270, Min(us): 1491, Max(us): 8595, 95th(us): 9000, 99th(us): 9000 [2024/04/26 19:20:05.116 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:05.224 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:05.302 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:05.396 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check diff failed 1-th time, retry later [2024/04/26 19:20:05.733 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:05.987 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:20:05 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362133901770366895 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:19:51.065365409 +0800 CST StartTs:449348997195235329 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e09950 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348997234556929} {CheckpointTs:449349000773500930 MinTableBarrierTs:449349001035382786 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/648906ec-6a65-483a-9abf-9c13694f5a73 {"id":"648906ec-6a65-483a-9abf-9c13694f5a73","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d387 648906ec-6a65-483a-9abf-9c13694f5a73 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362133901770366895,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:19:51.065365409+08:00","start-ts":449348997195235329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348997234556929} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349000773500930,"min-table-barrier-ts":449349001035382786,"admin-job-type":0} /tidb/cdc/default/default/task/position/648906ec-6a65-483a-9abf-9c13694f5a73/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362133901770366895 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:19:51.065365409 +0800 CST StartTs:449348997195235329 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e09950 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348997234556929} {CheckpointTs:449349000773500930 MinTableBarrierTs:449349001035382786 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/648906ec-6a65-483a-9abf-9c13694f5a73 {"id":"648906ec-6a65-483a-9abf-9c13694f5a73","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d387 648906ec-6a65-483a-9abf-9c13694f5a73 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362133901770366895,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:19:51.065365409+08:00","start-ts":449348997195235329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348997234556929} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349000773500930,"min-table-barrier-ts":449349001035382786,"admin-job-type":0} /tidb/cdc/default/default/task/position/648906ec-6a65-483a-9abf-9c13694f5a73/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362133901770366895 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:19:51.065365409 +0800 CST StartTs:449348997195235329 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e09950 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348997234556929} {CheckpointTs:449349000773500930 MinTableBarrierTs:449349001035382786 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349001035382788, checkpointTs: 449349000773500930, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/648906ec-6a65-483a-9abf-9c13694f5a73 {"id":"648906ec-6a65-483a-9abf-9c13694f5a73","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d387 648906ec-6a65-483a-9abf-9c13694f5a73 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362133901770366895,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:19:51.065365409+08:00","start-ts":449348997195235329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348997234556929} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349000773500930,"min-table-barrier-ts":449349001035382786,"admin-job-type":0} /tidb/cdc/default/default/task/position/648906ec-6a65-483a-9abf-9c13694f5a73/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/04/26 19:20:06.237 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check_ts_not_forward ddl-only-block-related-table table resourcecontrol.finish_mark not exists for 1-th check, retry later check diff failed 3-th time, retry later [2024/04/26 19:20:06.713 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:06.791 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table savepoint.finish_mark exists check diff successfully [2024/04/26 19:20:06.904 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:06.915 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:07.317 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] wait process cdc.test exit for 1-th time... [2024/04/26 19:20:07.472 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:07.581 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:07.785 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] wait process cdc.test exit for 2-th time... [2024/04/26 19:20:08.010 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff successfully wait process cdc.test exit for 1-th time... table resourcecontrol.finish_mark exists check diff successfully check diff failed 4-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:20:08 CST 2024] <<<<<< run test case savepoint success! >>>>>> wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... [2024/04/26 19:20:08.790 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:08.942 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:09.007 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] wait process cdc.test exit for 3-th time... [2024/04/26 19:20:09.292 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:20:09 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> [2024/04/26 19:20:09.577 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:20:09 CST 2024] <<<<<< run test case multi_capture success! >>>>>> [2024/04/26 19:20:09.823 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:10.167 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:10.205 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:10.569 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:10.604 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:10.690 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff failed 5-th time, retry later [2024/04/26 19:20:11.413 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:11.685 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:11.704 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:11.817 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [2024/04/26 19:20:12.181 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [2024/04/26 19:20:12.681 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:12.812 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] run task failed 1-th time, retry later [Pipeline] // withEnv [Pipeline] } check diff successfully [Pipeline] // node [Pipeline] } [2024/04/26 19:20:12.868 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 1m9.29830204s"] [2024/04/26 19:20:12.902 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [Pipeline] // podTemplate [Pipeline] } [2024/04/26 19:20:13.285 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [Pipeline] // withEnv wait process cdc.test exit for 1-th time... [2024/04/26 19:20:13.495 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 2-th time... [2024/04/26 19:20:14.067 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:14.081 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] check_ts_not_forward ddl-only-block-related-table wait process cdc.test exit for 3-th time... [2024/04/26 19:20:14.275 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:14.466 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:14.622 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:14.824 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:20:14 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> [2024/04/26 19:20:15.315 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:15.394 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:15.682 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:15.801 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:16.496 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:16.517 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:16.795 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:17.170 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:17.302 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:17.319 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:17.499 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:17.521 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:18.095 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:18.174 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:18.303 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:18.793 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:20:19.007 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:20:19.294 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:20:19.298 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:19.593 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:20:20.018 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:20:20.018 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [Fri Apr 26 19:20:17 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 1 [Fri Apr 26 19:20:17 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 2 [Fri Apr 26 19:20:17 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 3 [Fri Apr 26 19:20:17 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later [2024/04/26 19:20:20.211 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... [2024/04/26 19:20:20.729 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:20:21.044 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:20:21.190 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:20:21.400 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully [2024/04/26 19:20:21.593 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:20:21.991 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] table test.table10 not exists for 1-th check, retry later [2024/04/26 19:20:22.373 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:20:22.393 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:20:22.708 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:20:22.879 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:20:22.989 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community table test.finish_mark not exists for 1-th check, retry later Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 table test.finish_mark not exists for 2-th check, retry later UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... table test.finish_mark not exists for 3-th check, retry later Release Version: v7.5.1-5-g584533652 Edition: Community table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b table test.finish_mark not exists for 6-th check, retry later Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:20:23.577 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:20:23.881 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.table10 exists table test.table20 exists check diff successfully \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [2024/04/26 19:20:24.286 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:20:24.294 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:20:24.304 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [Pipeline] // cache \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [2024/04/26 19:20:24.921 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] // dir [Pipeline] } [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // withCredentials [Pipeline] // cache [Pipeline] } [Pipeline] } [2024/04/26 19:20:25.026 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.finish_mark not exists for 7-th check, retry later [Pipeline] // timeout [Pipeline] // dir [Pipeline] } [Pipeline] } [2024/04/26 19:20:25.370 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [Pipeline] // stage [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] } [Pipeline] // node [2024/04/26 19:20:25.893 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // withEnv [Pipeline] } [Pipeline] } [2024/04/26 19:20:26.180 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] // withEnv [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // podTemplate [2024/04/26 19:20:26.414 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/04/26 19:20:26.805 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:20:26.890 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:20:27.386 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:20:27.578 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table test.finish_mark not exists for 8-th check, retry later Starting Upstream TiDB... [2024/04/26 19:20:27.788 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:20:26.368 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`base_for_view`] [where="((TRUE) AND TRUE)"] ["source checksum"=4118227741] ["target checksum"=0] ["get source checksum cost"=1.660952ms] ["get target checksum cost"=1.040097ms] Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:20:28.499 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:20:28.974 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:20:28.985 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:20:29.302 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table test.finish_mark not exists for 9-th check, retry later [2024/04/26 19:20:29.668 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:20:30.194 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:20:30.376 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:20:30.435 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.finish_mark not exists for 10-th check, retry later [2024/04/26 19:20:31.410 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:20:31.496 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:20:31.721 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:20:32.206 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:20:32.496 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:20:32.704 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table test.finish_mark not exists for 11-th check, retry later [2024/04/26 19:20:33.387 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:20:33.596 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:20:33.885 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:20:34.909 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:20:35.081 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.finish_mark not exists for 12-th check, retry later [2024/04/26 19:20:35.820 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:20:35.905 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688adfd80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:14852, start at 2024-04-26 19:20:34.042823598 +0800 CST m=+6.228814332 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:34.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:34.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:34.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688adfd80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:14852, start at 2024-04-26 19:20:34.042823598 +0800 CST m=+6.228814332 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:34.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:34.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:34.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688ae0500013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:14924, start at 2024-04-26 19:20:34.108996612 +0800 CST m=+6.197869656 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:34.116 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:34.117 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:34.117 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:20:36.189 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:20:36.189 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:20:36.189 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:20:36.981 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.finish_mark not exists for 13-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:20:37.477 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.16137.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:20:38.574 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:20:39.181 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf22af609_997e_4830_9f66_1c24644ef4ca"] [2024/04/26 19:20:39.375 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs13a769ac_39d6_41b5_a123_84648bd36b11"] table test.finish_mark not exists for 14-th check, retry later [2024/04/26 19:20:39.573 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs491001cd_f446_449d_80c0_eb05adc1012c"] [2024/04/26 19:20:39.977 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscf53a836_d6d6_4477_8c10_aeb1b7557e1d"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + set +x + tso='449349009680105473 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349009680105473 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349009680105473 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:20:39 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1616616168.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:20:37.981 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`base_for_view`] [where="((TRUE) AND TRUE)"] [cost=11.612483883s] [2024/04/26 19:20:37.983 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [2024/04/26 19:20:40.326 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1b3aee6d_2f3b_41d1_9c5c_936ab7460c6c"] [2024/04/26 19:20:40.881 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1ef92c32_1301_458c_a62e_adfb54c23019"] [2024/04/26 19:20:40.972 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9acf84d8_a7c3_43b2_ab37_791fa474d41d"] table test.finish_mark not exists for 15-th check, retry later [2024/04/26 19:20:41.484 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs92174719_81ea_4131_8c5b_eadcc28fd7b5"] Starting Upstream TiDB... [2024/04/26 19:20:41.901 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... [2024/04/26 19:20:42.194 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:20:42.605 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:20:42.820 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:20:43.083 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:20:43 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d1cca53-fc36-46e1-85dd-a6bbc419bce1 {"id":"1d1cca53-fc36-46e1-85dd-a6bbc419bce1","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a228da1f7 1d1cca53-fc36-46e1-85dd-a6bbc419bce1 /tidb/cdc/default/default/upstream/7362134114104366151 {"id":7362134114104366151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d1cca53-fc36-46e1-85dd-a6bbc419bce1 {"id":"1d1cca53-fc36-46e1-85dd-a6bbc419bce1","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a228da1f7 1d1cca53-fc36-46e1-85dd-a6bbc419bce1 /tidb/cdc/default/default/upstream/7362134114104366151 {"id":7362134114104366151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d1cca53-fc36-46e1-85dd-a6bbc419bce1 {"id":"1d1cca53-fc36-46e1-85dd-a6bbc419bce1","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a228da1f7 1d1cca53-fc36-46e1-85dd-a6bbc419bce1 /tidb/cdc/default/default/upstream/7362134114104366151 {"id":7362134114104366151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349009680105473 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.16218.out cli changefeed create --start-ts=449349009680105473 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml [2024/04/26 19:20:43.177 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362134114104366151,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:20:43.462377148+08:00","start_ts":449349009680105473,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349009680105473,"checkpoint_ts":449349009680105473,"checkpoint_time":"2024-04-26 19:20:38.538"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 16-th check, retry later [2024/04/26 19:20:43.477 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:20:43.674 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:20:43.734 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2517 0 --:--:-- --:--:-- --:--:-- 2540 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:20:38.538","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:20:44.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:20:38.538","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:20:44.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:20:38.538","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:20:44.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-04-26 19:20:38.538' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:20:38.538","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:20:44.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:20:38.538","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:20:44.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-04-26 19:20:45' + echo 'sink_checkpoint_ts is 2024-04-26' 19:20:38.538 sink_checkpoint_ts is 2024-04-26 19:20:38.538 ++ date -d '2024-04-26 19:20:38.538' +%s + checkpoint_timestamp=1714130438 ++ date -d '2024-04-26 19:20:45' +%s + current_timestamp=1714130445 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [2024/04/26 19:20:44.892 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:20:45.099 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:20:45.189 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:20:45.206 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] table test.finish_mark exists check diff successfully [2024/04/26 19:20:45.310 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:20:45.426 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:20:45.615 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:20:45.674 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] wait process cdc.test exit for 1-th time... [2024/04/26 19:20:45.938 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... [2024/04/26 19:20:46.472 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:46.802 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:47.013 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:20:46 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/04/26 19:20:47.175 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:47.193 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:47.389 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:47.572 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:47.918 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table test.t1 exists + sleep 5 [2024/04/26 19:20:48.349 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:48.472 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:48.536 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:20:48.821 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:48.988 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:20:49.002 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:20:49.096 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:49.739 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:50.326 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:50.605 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:20:50.771 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:50.824 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:50.898 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:51.187 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:51.406 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:51.597 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:52.027 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:52.098 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:52.207 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:20:52.499 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688bf3c40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:7833, start at 2024-04-26 19:20:51.728833258 +0800 CST m=+9.652153827 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:51.820 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:51.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:51.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688bf3c40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:7833, start at 2024-04-26 19:20:51.728833258 +0800 CST m=+9.652153827 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:51.820 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:51.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:51.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688bf5480003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-nkv0q, pid:7867, start at 2024-04-26 19:20:51.815437362 +0800 CST m=+9.579138118 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:22:51.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:20:51.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:10:51.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... [2024/04/26 19:20:52.601 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:20:52.765 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:20:52.970 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2918 0 --:--:-- --:--:-- --:--:-- 2927 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:20:52.138","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:20:45.638","now_ts":"2024-04-26 19:20:52.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:20:52.138","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:20:45.638","now_ts":"2024-04-26' '19:20:52.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:20:52.138","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:20:45.638","now_ts":"2024-04-26' '19:20:52.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [2024/04/26 19:20:53.406 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:54.105 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:54.392 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:54.428 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:54.541 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:54.610 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.9034.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:20:55.084 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:55.375 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:55.622 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:55.875 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:56.087 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:56.105 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [Fri Apr 26 19:20:55 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 10 [Fri Apr 26 19:20:55 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 20 [Fri Apr 26 19:20:55 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later [2024/04/26 19:20:56.384 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:20:56.391 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:56.887 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:56.976 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:20:57.397 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + set +x + tso='449349014073114625 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + awk -F ' ' '{print $1}' + echo 449349014073114625 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + set +x [Fri Apr 26 19:20:57 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.90599061.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:20:57.592 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] table test.finish not exists for 2-th check, retry later [2024/04/26 19:20:57.971 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:58.190 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:58.466 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:58.515 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:58.970 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:58.995 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:59.072 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:59.195 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:20:59.685 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:59.784 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:20:59.794 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:20:59.906 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table test.finish exists check diff successfully wait process cdc.test exit for 1-th time... [2024/04/26 19:21:00.495 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:21:00 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73143509-fcb8-49ab-ad21-61a5164786cb {"id":"73143509-fcb8-49ab-ad21-61a5164786cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a22c501f8 73143509-fcb8-49ab-ad21-61a5164786cb /tidb/cdc/default/default/upstream/7362134173167174412 {"id":7362134173167174412,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73143509-fcb8-49ab-ad21-61a5164786cb {"id":"73143509-fcb8-49ab-ad21-61a5164786cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a22c501f8 73143509-fcb8-49ab-ad21-61a5164786cb /tidb/cdc/default/default/upstream/7362134173167174412 {"id":7362134173167174412,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73143509-fcb8-49ab-ad21-61a5164786cb {"id":"73143509-fcb8-49ab-ad21-61a5164786cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a22c501f8 73143509-fcb8-49ab-ad21-61a5164786cb /tidb/cdc/default/default/upstream/7362134173167174412 {"id":7362134173167174412,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.9096.out cli changefeed create --start-ts=449349014073114625 '--sink-uri=kafka://127.0.0.1:9092/ticdc-partition-table-test-13309?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/04/26 19:21:00.677 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Create changefeed successfully! ID: 5fa70079-28fe-4470-b66c-80d2f9350bf1 Info: {"upstream_id":7362134173167174412,"namespace":"default","id":"5fa70079-28fe-4470-b66c-80d2f9350bf1","sink_uri":"kafka://127.0.0.1:9092/ticdc-partition-table-test-13309?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:21:00.825931746+08:00","start_ts":449349014073114625,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349014073114625,"checkpoint_ts":449349014073114625,"checkpoint_time":"2024-04-26 19:20:55.296"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... [2024/04/26 19:21:00.974 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:00.980 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:21:01 CST 2024] <<<<<< run test case multi_topics success! >>>>>> [2024/04/26 19:21:01.481 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:01.585 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:01.681 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:01.066 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 12.46482732s"] [2024/04/26 19:21:01.974 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:02.072 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:02.080 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:02.275 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:02.277 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:02.682 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:02.770 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:02.772 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] + set +x [Fri Apr 26 19:21:02 CST 2024] <<<<<< START kafka consumer in partition_table case >>>>>> [2024/04/26 19:21:02.878 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:03.596 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:21:03.779 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:03.877 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:04.095 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:04.673 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:04.774 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:04.779 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:20:50.483 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/04/26 19:20:50.483 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7362133291850140132] [2024/04/26 19:20:50.484 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:20:50.484 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/04/26 19:20:50.484 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/04/26 19:20:50.484 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:20:50.486 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/04/26 19:20:50.486 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7362133291850140132] [2024/04/26 19:20:50.486 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:20:50.486 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/04/26 19:20:50.486 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/04/26 19:20:50.486 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/04/26 19:20:50.487 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/04/26 19:20:50.487 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/04/26 19:20:50.488 +08:00] [INFO] [store_cache.go:510] ["change store resolve state"] [store=2] [addr=127.0.0.1:20162] [from=unresolved] [to=resolved] [liveness-state=reachable] [2024/04/26 19:21:00.498 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/04/26 19:21:00.498 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/04/26 19:21:00.498 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/04/26 19:21:00.498 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/04/26 19:21:00.498 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/04/26 19:21:05.605 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:05.695 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:05.695 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:05.800 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 1-th check, retry later [2024/04/26 19:21:06.006 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:06.168 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:06.306 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:06.580 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:06.683 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:07.398 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:07.695 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:08.017 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:08.234 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table test.t2 not exists for 2-th check, retry later [2024/04/26 19:21:08.309 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:08.590 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:08.606 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] table big_txn.finish_mark_1 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:20:59 CST 2024] <<<<<< run test case big_txn success! >>>>>> [2024/04/26 19:21:09.610 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:10.069 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:10.070 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:10.095 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:10.123 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table test.t2 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:10.491 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:10.797 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:10.915 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:11.077 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:11.482 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:11.612 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688d239c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:20545, start at 2024-04-26 19:21:11.191827658 +0800 CST m=+5.502624592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:11.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:11.192 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:11.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 4-th check, retry later [2024/04/26 19:21:12.083 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:12.494 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:12.495 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:12.579 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:12.703 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:13.670 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:13.814 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 [2024/04/26 19:21:13.878 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.t2 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688d239c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:20545, start at 2024-04-26 19:21:11.191827658 +0800 CST m=+5.502624592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:11.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:11.192 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:11.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688d24400014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:20621, start at 2024-04-26 19:21:11.215146304 +0800 CST m=+5.428855059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:11.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:11.184 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:11.184 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:21:14.228 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:14.231 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:14.282 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:14.771 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later [2024/04/26 19:21:14.989 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:15.302 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:15.979 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.t2 not exists for 6-th check, retry later [2024/04/26 19:21:16.304 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.21859.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:21:16.883 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:17.080 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:21:17.281 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:17.295 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:17.407 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:21:16 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact The 1 times to try to start tidb cluster... [2024/04/26 19:21:17.569 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:17.595 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table partition_table.t2 not exists for 2-th check, retry later + set +x + tso='449349019695316993 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349019695316993 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:21:18 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2189121893.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:21:18.004 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:21:18.276 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.t2 not exists for 7-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:19.260 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later [2024/04/26 19:21:19.659 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.t2 not exists for 8-th check, retry later [2024/04/26 19:21:20.162 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6eeb4de6_baf6_44c6_811f_46fdff1f47a4"] [2024/04/26 19:21:20.361 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs011fbac2_119b_4174_995c_c484bfa8b6b8"] start tidb cluster in /tmp/tidb_cdc_test/avro_basic Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:20.812 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdaee0f01_9084_491c_b90c_6fa92da9e01e"] [2024/04/26 19:21:21.048 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6d7d689c_0ed6_4999_be04_5a600606f44d"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:21:21 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdfd8250-6205-4ce6-a61b-e89f99dd936a {"id":"cdfd8250-6205-4ce6-a61b-e89f99dd936a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23219f0a cdfd8250-6205-4ce6-a61b-e89f99dd936a /tidb/cdc/default/default/upstream/7362134275759198501 {"id":7362134275759198501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdfd8250-6205-4ce6-a61b-e89f99dd936a {"id":"cdfd8250-6205-4ce6-a61b-e89f99dd936a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23219f0a cdfd8250-6205-4ce6-a61b-e89f99dd936a /tidb/cdc/default/default/upstream/7362134275759198501 {"id":7362134275759198501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdfd8250-6205-4ce6-a61b-e89f99dd936a {"id":"cdfd8250-6205-4ce6-a61b-e89f99dd936a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23219f0a cdfd8250-6205-4ce6-a61b-e89f99dd936a /tidb/cdc/default/default/upstream/7362134275759198501 {"id":7362134275759198501,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table partition_table.finish_mark not exists for 2-th check, retry later [2024/04/26 19:21:21.498 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs64335520_bc3f_4a86_991e_4d4fbae365da"] pass check, checkpoint tso not forward after 10s run task successfully wait process 7752 exit for 1-th time... wait process 7752 exit for 2-th time... wait process 7752 exit for 3-th time... wait process 7752 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (7752) - No such process wait process 7752 exit for 5-th time... process 7752 already exit [Fri Apr 26 19:21:15 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.83178319.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:21:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362133901770366895 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:19:51.065365409 +0800 CST StartTs:449348997195235329 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00347ba70 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348997234556929} {CheckpointTs:449349001140502539 MinTableBarrierTs:449349019923120131 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b47fd11-9b65-4fb4-b0a7-17388e548458 {"id":"9b47fd11-9b65-4fb4-b0a7-17388e548458","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d476 9b47fd11-9b65-4fb4-b0a7-17388e548458 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362133901770366895,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:19:51.065365409+08:00","start-ts":449348997195235329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348997234556929} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349001140502539,"min-table-barrier-ts":449349020185264134,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b47fd11-9b65-4fb4-b0a7-17388e548458/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362133901770366895 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:19:51.065365409 +0800 CST StartTs:449348997195235329 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00347ba70 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348997234556929} {CheckpointTs:449349001140502539 MinTableBarrierTs:449349019923120131 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b47fd11-9b65-4fb4-b0a7-17388e548458 {"id":"9b47fd11-9b65-4fb4-b0a7-17388e548458","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d476 9b47fd11-9b65-4fb4-b0a7-17388e548458 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362133901770366895,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:19:51.065365409+08:00","start-ts":449348997195235329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348997234556929} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349001140502539,"min-table-barrier-ts":449349020185264134,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b47fd11-9b65-4fb4-b0a7-17388e548458/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7362133901770366895 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:19:51.065365409 +0800 CST StartTs:449348997195235329 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00347ba70 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449348997234556929} {CheckpointTs:449349001140502539 MinTableBarrierTs:449349019923120131 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449349001140502539, checkpointTs: 449349001140502539, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b47fd11-9b65-4fb4-b0a7-17388e548458 {"id":"9b47fd11-9b65-4fb4-b0a7-17388e548458","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a21c6d476 9b47fd11-9b65-4fb4-b0a7-17388e548458 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7362133901770366895,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-22183?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:19:51.065365409+08:00","start-ts":449348997195235329,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449348997234556929} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449349001140502539,"min-table-barrier-ts":449349020185264134,"admin-job-type":0} /tidb/cdc/default/default/task/position/9b47fd11-9b65-4fb4-b0a7-17388e548458/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362133901770366895 {"id":7362133901770366895,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later check diff failed 2-th time, retry later [2024/04/26 19:21:21.890 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf93a5091_92e7_48cc_ac8d_bc0841fa13bb"] [2024/04/26 19:21:21.905 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs72deabf1_c4e9_4c95_880b_ad445b9cb655"] table test.t2 not exists for 9-th check, retry later [2024/04/26 19:21:22.510 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs5c923eaf_f1ae_4e5c_9611_f356ebdc442a"] [Fri Apr 26 19:21:22 CST 2024] <<<<<< START kafka consumer in capture_session_done_during_task case >>>>>> lease 22318f1a23219f0a revoked check diff successfully check_ts_forward ddl-only-block-related-table Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:21:23.071 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:21:23.112 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 3-th check, retry later [2024/04/26 19:21:23.372 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:21:23.499 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table capture_session_done_during_task.t exists check diff failed 1-th time, retry later [2024/04/26 19:21:23.910 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] table test.t2 exists check diff successfully [2024/04/26 19:21:24.454 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] wait process cdc.test exit for 1-th time... [2024/04/26 19:21:24.790 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:21:24.827 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] changefeed is working normally rts: 449349021233840100->449349021495984100 checkpoint: 449349021233840100->449349021495984100 run task successfully Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:25.027 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688deffc001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:3856, start at 2024-04-26 19:21:24.271776878 +0800 CST m=+5.508103281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:24.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:24.273 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:24.273 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table partition_table.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... [2024/04/26 19:21:25.598 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:21:25.671 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:21:25.700 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:21:25.702 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:21:25 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> [2024/04/26 19:21:25.835 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later [2024/04/26 19:21:26.173 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:21:26.318 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:21:26 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688deffc001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:3856, start at 2024-04-26 19:21:24.271776878 +0800 CST m=+5.508103281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:24.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:24.273 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:24.273 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688df0a8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:3938, start at 2024-04-26 19:21:24.281640636 +0800 CST m=+5.426251000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:24.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:24.266 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:24.266 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:21:27.214 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table partition_table.finish_mark not exists for 5-th check, retry later [2024/04/26 19:21:27.325 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:21:27.380 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:27.908 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:21:27.913 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] check diff failed 3-th time, retry later [2024/04/26 19:21:24.023 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2024/04/26 19:21:28.674 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:21:28.927 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:21:29 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.51445146.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 6-th check, retry later [2024/04/26 19:21:29.595 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:21:29.668 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:21:29.704 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:21:29.711 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:21:29.809 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff successfully check diff failed 1-th time, retry later [2024/04/26 19:21:29.922 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:21:29.970 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:21:30.299 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:21:30.395 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:21:30.596 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:21:30.711 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 5 another is 4"] [2024/04/26 19:21:30.783 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:21:30.993 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688e57080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:9730, start at 2024-04-26 19:21:30.840091085 +0800 CST m=+5.799544458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:30.847 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:30.818 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:30.818 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:31.289 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:21:31.491 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table partition_table.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2024/04/26 19:21:32.278 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:21:32 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eded89a5-882b-438a-adcf-d0424b4da69a {"id":"eded89a5-882b-438a-adcf-d0424b4da69a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2354b5f3 eded89a5-882b-438a-adcf-d0424b4da69a /tidb/cdc/default/default/upstream/7362134333186869519 {"id":7362134333186869519,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eded89a5-882b-438a-adcf-d0424b4da69a {"id":"eded89a5-882b-438a-adcf-d0424b4da69a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2354b5f3 eded89a5-882b-438a-adcf-d0424b4da69a /tidb/cdc/default/default/upstream/7362134333186869519 {"id":7362134333186869519,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eded89a5-882b-438a-adcf-d0424b4da69a {"id":"eded89a5-882b-438a-adcf-d0424b4da69a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2354b5f3 eded89a5-882b-438a-adcf-d0424b4da69a /tidb/cdc/default/default/upstream/7362134333186869519 {"id":7362134333186869519,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2024/04/26 19:21:32.392 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:21:32.465 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check diff successfully [Fri Apr 26 19:21:32 CST 2024] <<<<<< START kafka consumer in changefeed_finish case >>>>>> [2024/04/26 19:21:32.812 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] wait process cdc.test exit for 1-th time... [2024/04/26 19:21:32.890 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:21:32.993 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:21:33.009 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688e57080012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:9730, start at 2024-04-26 19:21:30.840091085 +0800 CST m=+5.799544458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:30.847 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:30.818 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:30.818 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688e5a400013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:9803, start at 2024-04-26 19:21:31.054053442 +0800 CST m=+5.913957778 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:31.062 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:31.024 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:31.024 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... wait process cdc.test exit for 2-th time... Logging trace to /tmp/tidb_cdc_test/avro_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/avro_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:21:33.499 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:21:33.581 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table partition_table.finish_mark exists check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:21:33 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> [2024/04/26 19:21:33.792 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:21:33.924 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:21:34.091 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] wait process cdc.test exit for 1-th time... [2024/04/26 19:21:34.410 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:21:34.627 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later [2024/04/26 19:21:35.081 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] wait process cdc.test exit for 3-th time... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 372 387 --:--:-- --:--:-- --:--:-- 390 {"compatibility":"NONE"}+ pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.11000.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:21:35.694 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:21:35 CST 2024] <<<<<< run test case partition_table success! >>>>>> [2024/04/26 19:21:36.294 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:21:36.300 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:21:36.672 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:21:36.674 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:21:36.784 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:21:36.870 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:21:37.108 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:21:37.190 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] + set +x + tso='449349024695713795 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349024695713795 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:21:37 CST 2024] <<<<<< START cdc server in avro_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.1103811040.out server --log-file /tmp/tidb_cdc_test/avro_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/avro_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully [2024/04/26 19:21:37.683 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:21:37.875 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:21:38.167 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:21:38.283 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: kafka... <<================= [2024/04/26 19:21:38.672 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:21:38.681 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] The 1 times to try to start tidb cluster... [2024/04/26 19:21:39.380 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:21:40.309 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:21:40.598 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:21:40 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8b38c22-960f-44ac-b632-56818b6b9b68 {"id":"e8b38c22-960f-44ac-b632-56818b6b9b68","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a236d1bf3 e8b38c22-960f-44ac-b632-56818b6b9b68 /tidb/cdc/default/default/upstream/7362134358190283887 {"id":7362134358190283887,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8b38c22-960f-44ac-b632-56818b6b9b68 {"id":"e8b38c22-960f-44ac-b632-56818b6b9b68","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a236d1bf3 e8b38c22-960f-44ac-b632-56818b6b9b68 /tidb/cdc/default/default/upstream/7362134358190283887 {"id":7362134358190283887,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8b38c22-960f-44ac-b632-56818b6b9b68 {"id":"e8b38c22-960f-44ac-b632-56818b6b9b68","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a236d1bf3 e8b38c22-960f-44ac-b632-56818b6b9b68 /tidb/cdc/default/default/upstream/7362134358190283887 {"id":7362134358190283887,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.11085.out cli changefeed create --start-ts=449349024695713795 '--sink-uri=kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true&avro-decimal-handling-mode=string&avro-bigint-unsigned-handling-mode=string' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: 424e57fc-f0fa-4c21-94ee-e7685344fe38 Info: {"upstream_id":7362134358190283887,"namespace":"default","id":"424e57fc-f0fa-4c21-94ee-e7685344fe38","sink_uri":"kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true\u0026avro-decimal-handling-mode=string\u0026avro-bigint-unsigned-handling-mode=string","create_time":"2024-04-26T19:21:40.745023059+08:00","start_ts":449349024695713795,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349024695713795,"checkpoint_ts":449349024695713795,"checkpoint_time":"2024-04-26 19:21:35.818"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:21:40.975 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:21:41.183 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:21:41.573 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:21:41.596 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:41.779 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:21:41.978 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:21:42.077 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] + set +x [Fri Apr 26 19:21:42 CST 2024] <<<<<< START kafka consumer in avro_basic case >>>>>> schema registry uri found: http://127.0.0.1:8088 [2024/04/26 19:21:41.385 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] ["source checksum"=0] ["target checksum"=1674903914] ["get source checksum cost"=1.623426ms] ["get target checksum cost"=2.871625ms] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:21:42.882 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:21:42.888 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:21:42.969 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:42.977 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:42.982 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:21:43.392 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:21:43.679 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table test.finish_mark not exists for 1-th check, retry later [2024/04/26 19:21:44.191 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:44.980 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:44.988 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:44.995 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:45.493 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/04/26 19:21:45.583 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:45.674 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:45.983 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:21:46.067 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.finish_mark not exists for 2-th check, retry later table test1.finishmark exists [2024/04/26 19:21:46.182 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/04/26 19:21:46.184 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/04/26 19:21:46.186 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/04/26 19:21:46.191 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/04/26 19:21:46.192 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/04/26 19:21:46.192 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=13.491091ms] [2024/04/26 19:21:46.195 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/04/26 19:21:46.197 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/04/26 19:21:46.198 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/04/26 19:21:46.203 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/04/26 19:21:46.205 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/04/26 19:21:46.205 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=12.644707ms] [2024/04/26 19:21:46.205 +08:00] [INFO] [main.go:95] ["compare checksum passed"] [2024/04/26 19:21:46.583 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] wait process cdc.test exit for 1-th time... [2024/04/26 19:21:46.780 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:46.782 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:46.898 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:21:47.000 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:21:47.002 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] wait process cdc.test exit for 2-th time... [2024/04/26 19:21:47.276 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:47.579 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:21:47 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> table test.finish_mark exists [2024/04/26 19:21:48.179 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] check diff successfully start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:21:48.291 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:21:48.575 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:48.791 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] wait process cdc.test exit for 1-th time... [2024/04/26 19:21:48.908 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] wait process cdc.test exit for 2-th time... [2024/04/26 19:21:49.096 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:49.184 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:21:49 CST 2024] <<<<<< run test case avro_basic success! >>>>>> [2024/04/26 19:21:50.369 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:50.477 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:50.798 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:50.891 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:50.905 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:50.988 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:21:51.108 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:51.203 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688f8abc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:20954, start at 2024-04-26 19:21:50.513879767 +0800 CST m=+5.413055689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:50.521 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:50.511 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:50.511 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:51.397 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:21:51.996 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:52.066 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:52.670 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:52.873 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:53.273 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/04/26 19:21:53.279 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:53.579 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688f8abc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:20954, start at 2024-04-26 19:21:50.513879767 +0800 CST m=+5.413055689 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:50.521 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:50.511 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:50.511 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688f8c3c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:21038, start at 2024-04-26 19:21:50.610485485 +0800 CST m=+5.364510070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:50.617 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:50.607 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:50.607 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:21:52.183 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/04/26 19:21:52.500 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:21:52.509 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:21:52.756 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:21:52.759 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:21:52.767 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:21:52.769 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [2024/04/26 19:21:53.994 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:21:54.015 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:21:54.206 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [Pipeline] // cache [2024/04/26 19:21:54.090 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [Pipeline] } [Pipeline] // dir [Pipeline] } [2024/04/26 19:21:54.215 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:21:54.226 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:21:54.229 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:21:54.454 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [2024/04/26 19:21:54.816 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:54.913 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2024/04/26 19:21:54.997 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:55.169 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [2024/04/26 19:21:55.333 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:55.406 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.22268.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:21:55.693 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:55.571 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:21:55.808 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:21:55.815 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:55.851 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:21:55.854 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:21:56.080 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:56.199 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/04/26 19:21:56.400 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:56.867 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:57.076 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/04/26 19:21:57.169 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] + set +x + tso='449349029937020929 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + awk -F ' ' '{print $1}' + echo 449349029937020929 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + set +x ***************** properties ***************** "dotransactions"="false" "recordcount"="10000" "mysql.host"="127.0.0.1" "operationcount"="0" "workload"="core" "readallfields"="true" "readproportion"="0" "threadcount"="10" "mysql.user"="root" "insertproportion"="0" "mysql.db"="move_table" "mysql.port"="4000" "requestdistribution"="uniform" "scanproportion"="0" "updateproportion"="0" ********************************************** [2024/04/26 19:21:57.379 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688feb3c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:9509, start at 2024-04-26 19:21:56.701127025 +0800 CST m=+5.523803747 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:56.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:56.687 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:56.687 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:57.904 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:58.085 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:58.466 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:58.680 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:58.800 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:21:59.094 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:59.174 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688feb3c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:9509, start at 2024-04-26 19:21:56.701127025 +0800 CST m=+5.523803747 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:56.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:56.687 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:56.687 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c688febf40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-fd8mf, pid:9591, start at 2024-04-26 19:21:56.735476261 +0800 CST m=+5.410426884 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:56.744 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:56.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:56.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:21:59.612 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:21:59.804 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/04/26 19:22:00.177 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:22:00.186 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:22:00.691 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Run finished, takes 2.93294623s INSERT - Takes(s): 2.9, Count: 10000, OPS: 3416.7, Avg(us): 2887, Min(us): 707, Max(us): 62400, 95th(us): 3000, 99th(us): 58000 [Fri Apr 26 19:22:00 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2235122353.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:22:00.782 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/04/26 19:22:01.135 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689018f4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:23180, start at 2024-04-26 19:21:59.624504006 +0800 CST m=+6.205654259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:59.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:59.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:59.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689018f4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:23180, start at 2024-04-26 19:21:59.624504006 +0800 CST m=+6.205654259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:59.631 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:59.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:59.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68901b280005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-6lh0m, pid:23254, start at 2024-04-26 19:21:59.757856933 +0800 CST m=+6.189175837 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:23:59.769 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:21:59.754 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:11:59.754 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ***************** properties ***************** "mysql.host"="127.0.0.1" "threadcount"="4" "scanproportion"="0" "recordcount"="20" "mysql.db"="changefeed_auto_stop_1" "operationcount"="0" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" "workload"="core" "dotransactions"="false" "readproportion"="0" "mysql.user"="root" "mysql.port"="4000" "requestdistribution"="uniform" ********************************************** Run finished, takes 8.837736ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3782.1, Avg(us): 1522, Min(us): 801, Max(us): 3730, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "threadcount"="4" "updateproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.user"="root" "requestdistribution"="uniform" "readallfields"="true" "workload"="core" "mysql.port"="4000" "readproportion"="0" "insertproportion"="0" "scanproportion"="0" "operationcount"="0" "mysql.db"="changefeed_auto_stop_2" "recordcount"="20" ********************************************** Run finished, takes 8.934425ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4012.3, Avg(us): 1693, Min(us): 1010, Max(us): 3954, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "requestdistribution"="uniform" "scanproportion"="0" "mysql.db"="changefeed_auto_stop_3" "threadcount"="4" "mysql.host"="127.0.0.1" "recordcount"="20" "updateproportion"="0" "readallfields"="true" "readproportion"="0" "insertproportion"="0" "operationcount"="0" "workload"="core" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" ********************************************** Run finished, takes 8.571248ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4600.9, Avg(us): 1628, Min(us): 836, Max(us): 4153, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "mysql.db"="changefeed_auto_stop_4" "scanproportion"="0" "mysql.port"="4000" "threadcount"="4" "workload"="core" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" "readproportion"="0" "operationcount"="0" "updateproportion"="0" "requestdistribution"="uniform" "recordcount"="20" "insertproportion"="0" "readallfields"="true" ********************************************** Run finished, takes 9.586151ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3741.1, Avg(us): 1780, Min(us): 958, Max(us): 4301, 95th(us): 5000, 99th(us): 5000 [Fri Apr 26 19:22:02 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1095610958.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh: line 1: 15104 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 (wd: /tmp/tidb_cdc_test/kafka_column_selector) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 [2024/04/26 19:22:02.598 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:22:02.788 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:22:02.887 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:22:02.977 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/04/26 19:22:03.388 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:22:03.470 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/04/26 19:22:02.700 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/04/26 19:22:02.994 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:22:03.004 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:22:03.234 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:22:03.245 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:22:03.260 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:22:03.263 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:22:03.498 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:22:03.511 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:22:03.722 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:22:03.724 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:22:03.734 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:22:03.737 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:03 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.22403.out cli changefeed create --start-ts=449349029937020929 '--sink-uri=kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 06545d08-253d-4d31-9238-c07b879f9f8b Info: {"upstream_id":7362134449908916220,"namespace":"default","id":"06545d08-253d-4d31-9238-c07b879f9f8b","sink_uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:22:03.881944966+08:00","start_ts":449349029937020929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349029937020929,"checkpoint_ts":449349029937020929,"checkpoint_time":"2024-04-26 19:21:55.812"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:22:03.971 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:22:03.997 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:22:04.203 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:22:04.210 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24477.out cli tso query --pd=http://127.0.0.1:2379 [2024/04/26 19:22:04.228 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:22:04.234 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:22:04.435 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:22:04.462 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:04 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 338 352 --:--:-- --:--:-- --:--:-- 357 {"compatibility":"NONE"}The 1 times to try to start tidb cluster... [2024/04/26 19:22:04.676 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:22:04.688 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:22:04.700 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:22:04.713 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:22:04.947 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:22:04.968 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] start tidb cluster in /tmp/tidb_cdc_test/canal_json_handle_key_only Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:22:05.211 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:22:05.222 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:22:05.228 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:22:05.237 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:22:05.128 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] + set +x [Fri Apr 26 19:22:05 CST 2024] <<<<<< START kafka consumer in move_table case >>>>>> [Fri Apr 26 19:22:05 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2244222444.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:22:05.418 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:22:05.434 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] + set +x + tso='449349032086863873 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349032086863873 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:22:05 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2450524507.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:22:05.445 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfcc83a58_f60a_4728_817c_a55a798a2abe"] [2024/04/26 19:22:05.617 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:22:05.627 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 19:22:05.633 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:22:05.642 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:05 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac9c4b48-3898-49e9-939d-92597792e38f {"id":"ac9c4b48-3898-49e9-939d-92597792e38f","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d1633f ac9c4b48-3898-49e9-939d-92597792e38f /tidb/cdc/default/default/upstream/7362134475887907076 {"id":7362134475887907076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac9c4b48-3898-49e9-939d-92597792e38f {"id":"ac9c4b48-3898-49e9-939d-92597792e38f","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d1633f ac9c4b48-3898-49e9-939d-92597792e38f /tidb/cdc/default/default/upstream/7362134475887907076 {"id":7362134475887907076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac9c4b48-3898-49e9-939d-92597792e38f {"id":"ac9c4b48-3898-49e9-939d-92597792e38f","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d1633f ac9c4b48-3898-49e9-939d-92597792e38f /tidb/cdc/default/default/upstream/7362134475887907076 {"id":7362134475887907076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:22:05 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1101611018.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:22:05.831 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:22:05.859 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 19:22:06.034 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:22:06.044 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 19:22:06.065 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:22:06.076 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:22:06.236 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 19:22:06.260 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:22:06.461 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/04/26 19:22:06.472 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/04/26 19:22:06.482 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/04/26 19:22:06.492 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/04/26 19:22:06.354 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc14ce61c_ea25_4fce_b0bd_3304b7081184"] [2024/04/26 19:22:06.732 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/04/26 19:22:06.756 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/04/26 19:22:06.643 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs32464c85_a1fa_4629_a1ed_1db4b28c4b4d"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:22:06.902 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs56df3c77_0274_4644_bbdd_77f1811a2721"] [2024/04/26 19:22:07.049 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9519396e_822b_4e14_976f_c15cd2f2dc29"] Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:22:07.144 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/04/26 19:22:07.160 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/04/26 19:22:07.167 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/04/26 19:22:07.178 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/04/26 19:22:07.415 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/04/26 19:22:07.436 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/04/26 19:22:07.653 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/04/26 19:22:07.666 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/04/26 19:22:07.668 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/04/26 19:22:08.395 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/06545d08-253d-4d31-9238-c07b879f9f8b {UpstreamID:7362134449908916220 Namespace:default ID:06545d08-253d-4d31-9238-c07b879f9f8b SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:22:03.881944966 +0800 CST StartTs:449349029937020929 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0020d13b0 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449349032047280131} {CheckpointTs:449349030461309141 MinTableBarrierTs:449349032938569730 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ee63afb4-6e20-4162-ba19-aae55f656c44 {"id":"ee63afb4-6e20-4162-ba19-aae55f656c44","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b252 ee63afb4-6e20-4162-ba19-aae55f656c44 /tidb/cdc/default/default/changefeed/info/06545d08-253d-4d31-9238-c07b879f9f8b {"upstream-id":7362134449908916220,"namespace":"default","changefeed-id":"06545d08-253d-4d31-9238-c07b879f9f8b","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:22:03.881944966+08:00","start-ts":449349029937020929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449349032047280131} /tidb/cdc/default/default/changefeed/status/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":449349030461309141,"min-table-barrier-ts":449349032938569730,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eedbf0c-43f3-4e6c-abda-458f3dafd31b/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee63afb4-6e20-4162-ba19-aae55f656c44/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/06545d08-253d-4d31-9238-c07b879f9f8b {UpstreamID:7362134449908916220 Namespace:default ID:06545d08-253d-4d31-9238-c07b879f9f8b SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:22:03.881944966 +0800 CST StartTs:449349029937020929 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0020d13b0 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449349032047280131} {CheckpointTs:449349030461309141 MinTableBarrierTs:449349032938569730 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ee63afb4-6e20-4162-ba19-aae55f656c44 {"id":"ee63afb4-6e20-4162-ba19-aae55f656c44","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b252 ee63afb4-6e20-4162-ba19-aae55f656c44 /tidb/cdc/default/default/changefeed/info/06545d08-253d-4d31-9238-c07b879f9f8b {"upstream-id":7362134449908916220,"namespace":"default","changefeed-id":"06545d08-253d-4d31-9238-c07b879f9f8b","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:22:03.881944966+08:00","start-ts":449349029937020929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449349032047280131} /tidb/cdc/default/default/changefeed/status/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":449349030461309141,"min-table-barrier-ts":449349032938569730,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eedbf0c-43f3-4e6c-abda-458f3dafd31b/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee63afb4-6e20-4162-ba19-aae55f656c44/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/06545d08-253d-4d31-9238-c07b879f9f8b {UpstreamID:7362134449908916220 Namespace:default ID:06545d08-253d-4d31-9238-c07b879f9f8b SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:22:03.881944966 +0800 CST StartTs:449349029937020929 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0020d13b0 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449349032047280131} {CheckpointTs:449349030461309141 MinTableBarrierTs:449349032938569730 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ee63afb4-6e20-4162-ba19-aae55f656c44 {"id":"ee63afb4-6e20-4162-ba19-aae55f656c44","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b252 ee63afb4-6e20-4162-ba19-aae55f656c44 /tidb/cdc/default/default/changefeed/info/06545d08-253d-4d31-9238-c07b879f9f8b {"upstream-id":7362134449908916220,"namespace":"default","changefeed-id":"06545d08-253d-4d31-9238-c07b879f9f8b","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:22:03.881944966+08:00","start-ts":449349029937020929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449349032047280131} /tidb/cdc/default/default/changefeed/status/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":449349030461309141,"min-table-barrier-ts":449349032938569730,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eedbf0c-43f3-4e6c-abda-458f3dafd31b/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee63afb4-6e20-4162-ba19-aae55f656c44/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127+ grep -q 'etcd info' .0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 26 19:22:08 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2255322555.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:22:08.624 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:08 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/340ca098-3976-4696-816c-612d7367f8a2 {"id":"340ca098-3976-4696-816c-612d7367f8a2","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23da11f4 340ca098-3976-4696-816c-612d7367f8a2 /tidb/cdc/default/default/upstream/7362134485012922065 {"id":7362134485012922065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/340ca098-3976-4696-816c-612d7367f8a2 {"id":"340ca098-3976-4696-816c-612d7367f8a2","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23da11f4 340ca098-3976-4696-816c-612d7367f8a2 /tidb/cdc/default/default/upstream/7362134485012922065 {"id":7362134485012922065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/340ca098-3976-4696-816c-612d7367f8a2 {"id":"340ca098-3976-4696-816c-612d7367f8a2","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23da11f4 340ca098-3976-4696-816c-612d7367f8a2 /tidb/cdc/default/default/upstream/7362134485012922065 {"id":7362134485012922065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.24548.out cli changefeed create --start-ts=449349032086863873 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-22508?protocol=open-protocol&partition-num=4&kafka-version=2.4.1' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:08 GMT < Content-Length: 867 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a74454f3-16be-452b-b1e9-310b773d3569 {"id":"a74454f3-16be-452b-b1e9-310b773d3569","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ac9c4b48-3898-49e9-939d-92597792e38f {"id":"ac9c4b48-3898-49e9-939d-92597792e38f","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d1633f ac9c4b48-3898-49e9-939d-92597792e38f /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d16366 a74454f3-16be-452b-b1e9-310b773d3569 /tidb/cdc/default/default/upstream/7362134475887907076 {"id":7362134475887907076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a74454f3-16be-452b-b1e9-310b773d3569 {"id":"a74454f3-16be-452b-b1e9-310b773d3569","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ac9c4b48-3898-49e9-939d-92597792e38f {"id":"ac9c4b48-3898-49e9-939d-92597792e38f","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d1633f ac9c4b48-3898-49e9-939d-92597792e38f /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d16366 a74454f3-16be-452b-b1e9-310b773d3569 /tidb/cdc/default/default/upstream/7362134475887907076 {"id":7362134475887907076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a74454f3-16be-452b-b1e9-310b773d3569 {"id":"a74454f3-16be-452b-b1e9-310b773d3569","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ac9c4b48-3898-49e9-939d-92597792e38f {"id":"ac9c4b48-3898-49e9-939d-92597792e38f","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d1633f ac9c4b48-3898-49e9-939d-92597792e38f /tidb/cdc/default/__cdc_meta__/owner/22318f1a23d16366 a74454f3-16be-452b-b1e9-310b773d3569 /tidb/cdc/default/default/upstream/7362134475887907076 {"id":7362134475887907076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 26 19:22:08 CST 2024] <<<<<< START kafka consumer in changefeed_auto_stop case >>>>>> check_changefeed_state http://127.0.0.1:2379 4f8601ed-ac22-4c8b-bacc-53df43fbf80e normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=4f8601ed-ac22-4c8b-bacc-53df43fbf80e + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 4f8601ed-ac22-4c8b-bacc-53df43fbf80e -s + info='{ "upstream_id": 7362134475887907076, "namespace": "default", "id": "4f8601ed-ac22-4c8b-bacc-53df43fbf80e", "state": "normal", "checkpoint_tso": 449349031464009733, "checkpoint_time": "2024-04-26 19:22:01.637", "error": null }' + echo '{ "upstream_id": 7362134475887907076, "namespace": "default", "id": "4f8601ed-ac22-4c8b-bacc-53df43fbf80e", "state": "normal", "checkpoint_tso": 449349031464009733, "checkpoint_time": "2024-04-26 19:22:01.637", "error": null }' { "upstream_id": 7362134475887907076, "namespace": "default", "id": "4f8601ed-ac22-4c8b-bacc-53df43fbf80e", "state": "normal", "checkpoint_tso": 449349031464009733, "checkpoint_time": "2024-04-26 19:22:01.637", "error": null } ++ echo '{' '"upstream_id":' 7362134475887907076, '"namespace":' '"default",' '"id":' '"4f8601ed-ac22-4c8b-bacc-53df43fbf80e",' '"state":' '"normal",' '"checkpoint_tso":' 449349031464009733, '"checkpoint_time":' '"2024-04-26' '19:22:01.637",' '"error":' null '}' ++ jq -r .state Create changefeed successfully! ID: c2d5857d-eebf-42bd-a9f9-32b8a9428528 Info: {"upstream_id":7362134485012922065,"namespace":"default","id":"c2d5857d-eebf-42bd-a9f9-32b8a9428528","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-22508?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:22:08.915065542+08:00","start_ts":449349032086863873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349032086863873,"checkpoint_ts":449349032086863873,"checkpoint_time":"2024-04-26 19:22:04.013"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362134475887907076, '"namespace":' '"default",' '"id":' '"4f8601ed-ac22-4c8b-bacc-53df43fbf80e",' '"state":' '"normal",' '"checkpoint_tso":' 449349031464009733, '"checkpoint_time":' '"2024-04-26' '19:22:01.637",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later [2024/04/26 19:22:09.268 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:22:09.309 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:22:09.403 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector_avro Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:22:09.792 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:22:09.821 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:22:09.826 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/04/26 19:22:09.927 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/04/26 19:22:10.135 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] + set +x [Fri Apr 26 19:22:10 CST 2024] <<<<<< START kafka consumer in ddl_attributes case >>>>>> table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:22:11.317 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:22:11.812 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/06545d08-253d-4d31-9238-c07b879f9f8b {UpstreamID:7362134449908916220 Namespace:default ID:06545d08-253d-4d31-9238-c07b879f9f8b SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:22:03.881944966 +0800 CST StartTs:449349029937020929 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0032c0240 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449349032047280131} {CheckpointTs:449349030998442029 MinTableBarrierTs:449349034000252933 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/b169abfd-a008-4a56-9991-9ce86630c87e {"id":"b169abfd-a008-4a56-9991-9ce86630c87e","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ee63afb4-6e20-4162-ba19-aae55f656c44 {"id":"ee63afb4-6e20-4162-ba19-aae55f656c44","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b252 ee63afb4-6e20-4162-ba19-aae55f656c44 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b276 b169abfd-a008-4a56-9991-9ce86630c87e /tidb/cdc/default/default/changefeed/info/06545d08-253d-4d31-9238-c07b879f9f8b {"upstream-id":7362134449908916220,"namespace":"default","changefeed-id":"06545d08-253d-4d31-9238-c07b879f9f8b","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:22:03.881944966+08:00","start-ts":449349029937020929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449349032047280131} /tidb/cdc/default/default/changefeed/status/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":449349030998442029,"min-table-barrier-ts":449349034000252933,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eedbf0c-43f3-4e6c-abda-458f3dafd31b/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b169abfd-a008-4a56-9991-9ce86630c87e/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee63afb4-6e20-4162-ba19-aae55f656c44/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/06545d08-253d-4d31-9238-c07b879f9f8b {UpstreamID:7362134449908916220 Namespace:default ID:06545d08-253d-4d31-9238-c07b879f9f8b SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:22:03.881944966 +0800 CST StartTs:449349029937020929 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0032c0240 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449349032047280131} {CheckpointTs:449349030998442029 MinTableBarrierTs:449349034000252933 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/b169abfd-a008-4a56-9991-9ce86630c87e {"id":"b169abfd-a008-4a56-9991-9ce86630c87e","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ee63afb4-6e20-4162-ba19-aae55f656c44 {"id":"ee63afb4-6e20-4162-ba19-aae55f656c44","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b252 ee63afb4-6e20-4162-ba19-aae55f656c44 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b276 b169abfd-a008-4a56-9991-9ce86630c87e /tidb/cdc/default/default/changefeed/info/06545d08-253d-4d31-9238-c07b879f9f8b {"upstream-id":7362134449908916220,"namespace":"default","changefeed-id":"06545d08-253d-4d31-9238-c07b879f9f8b","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:22:03.881944966+08:00","start-ts":449349029937020929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449349032047280131} /tidb/cdc/default/default/changefeed/status/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":449349030998442029,"min-table-barrier-ts":449349034000252933,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eedbf0c-43f3-4e6c-abda-458f3dafd31b/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b169abfd-a008-4a56-9991-9ce86630c87e/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee63afb4-6e20-4162-ba19-aae55f656c44/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/06545d08-253d-4d31-9238-c07b879f9f8b {UpstreamID:7362134449908916220 Namespace:default ID:06545d08-253d-4d31-9238-c07b879f9f8b SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-04-26 19:22:03.881944966 +0800 CST StartTs:449349029937020929 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0032c0240 State:normal Error: Warning: CreatorVersion:v7.5.1-57-g980f60a5b Epoch:449349032047280131} {CheckpointTs:449349030998442029 MinTableBarrierTs:449349034000252933 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9eedbf0c-43f3-4e6c-abda-458f3dafd31b {"id":"9eedbf0c-43f3-4e6c-abda-458f3dafd31b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/b169abfd-a008-4a56-9991-9ce86630c87e {"id":"b169abfd-a008-4a56-9991-9ce86630c87e","address":"127.0.0.1:8302","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/capture/ee63afb4-6e20-4162-ba19-aae55f656c44 {"id":"ee63afb4-6e20-4162-ba19-aae55f656c44","address":"127.0.0.1:8301","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b20b 9eedbf0c-43f3-4e6c-abda-458f3dafd31b /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b252 ee63afb4-6e20-4162-ba19-aae55f656c44 /tidb/cdc/default/__cdc_meta__/owner/22318f1a23b7b276 b169abfd-a008-4a56-9991-9ce86630c87e /tidb/cdc/default/default/changefeed/info/06545d08-253d-4d31-9238-c07b879f9f8b {"upstream-id":7362134449908916220,"namespace":"default","changefeed-id":"06545d08-253d-4d31-9238-c07b879f9f8b","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-15981?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-04-26T19:22:03.881944966+08:00","start-ts":449349029937020929,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-57-g980f60a5b","epoch":449349032047280131} /tidb/cdc/default/default/changefeed/status/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":449349030998442029,"min-table-barrier-ts":449349034000252933,"admin-job-type":0} /tidb/cdc/default/default/task/position/9eedbf0c-43f3-4e6c-abda-458f3dafd31b/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b169abfd-a008-4a56-9991-9ce86630c87e/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ee63afb4-6e20-4162-ba19-aae55f656c44/06545d08-253d-4d31-9238-c07b879f9f8b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7362134449908916220 {"id":7362134449908916220,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table move_table.usertable exists go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/benbjohnson/clock v1.3.5 [2024/04/26 19:22:11.913 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20230619063055-fd67d94318fd go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/philhofer/fwd v1.1.1 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [2024/04/26 19:22:12.092 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c [2024/04/26 19:22:12.705 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/04/26 19:22:12.796 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading go.etcd.io/bbolt v1.3.8 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 [2024/04/26 19:22:12.927 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 [2024/04/26 19:22:13.277 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/04/26 19:22:13.319 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later [2024/04/26 19:22:13.441 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2024/04/26 19:22:14.175 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:22:13.618 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/04/26 19:22:13.871 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:22:13.929 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:22:14.085 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:22:14.088 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:22:14.162 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:22:14.165 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:22:14.375 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:22:14.432 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:22:14.714 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:22:14.731 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:22:14.712 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:22:14.713 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:22:14.792 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:22:14.795 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:22:14.915 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:22:15.089 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:22:15.277 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/04/26 19:22:15.283 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later [2024/04/26 19:22:15.593 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:22:15.594 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:22:15.465 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] go: downloading github.com/ardielle/ardielle-go v1.5.2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:22:15.754 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:22:15.757 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:22:15.728 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:22:15.893 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:22:15.925 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:22:16.012 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:22:15.989 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/04/26 19:22:15.990 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:22:16.182 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:22:16.183 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:22:16.281 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:22:16.284 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:22:16.438 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:22:16.523 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:22:16.688 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:22:16.810 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:22:16.811 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:22:16.904 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:22:16.906 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later [2024/04/26 19:22:17.256 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:22:17.402 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689108cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:12067, start at 2024-04-26 19:22:15.006734054 +0800 CST m=+5.878573444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:24:15.014 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:22:15.013 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:12:15.013 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689108cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:12067, start at 2024-04-26 19:22:15.006734054 +0800 CST m=+5.878573444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:24:15.014 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:22:15.013 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:12:15.013 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68910a70000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:12148, start at 2024-04-26 19:22:15.087681061 +0800 CST m=+5.842044797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:24:15.096 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:22:15.068 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:12:15.068 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:22:17.519 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:22:17.571 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:22:17.601 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:22:17.687 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 19:22:17.688 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:22:17.801 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:22:17.770 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:22:17.772 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 19:22:18.091 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/04/26 19:22:18.000 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:22:18.089 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 19:22:18.228 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/04/26 19:22:18.229 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:22:18.316 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:22:18.318 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:22:18.474 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 19:22:18.586 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:22:18.610 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:22:18.774 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:22:19.104 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/04/26 19:22:19.396 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:22:19.420 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff successfully wait process cdc.test exit for 1-th time... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.13419.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... [2024/04/26 19:22:20.312 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:22:20.482 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689151500013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:28554, start at 2024-04-26 19:22:19.642415628 +0800 CST m=+5.449129651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:24:19.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:22:19.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:12:19.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/04/26 19:22:20.705 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/04/26 19:22:20.785 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:22:20.981 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:22:20 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> + set +x + tso='449349036241846273 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349036241846273 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:22:21 CST 2024] <<<<<< START cdc server in canal_json_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.1344913451.out server --log-file /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:22:21.568 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:22:21.584 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:22:22.070 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/04/26 19:22:22.081 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:22:22.372 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689151500013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:28554, start at 2024-04-26 19:22:19.642415628 +0800 CST m=+5.449129651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:24:19.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:22:19.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:12:19.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689152c80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-211v9, pid:28648, start at 2024-04-26 19:22:19.703517901 +0800 CST m=+5.372268484 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:24:19.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:22:19.698 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:12:19.698 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/04/26 19:22:23.146 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:22:23.473 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:22:23.778 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:22:23.779 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/04/26 19:22:24.166 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:24 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15295f1d-ef02-49e8-8d1d-2007c626d4db {"id":"15295f1d-ef02-49e8-8d1d-2007c626d4db","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24195af5 15295f1d-ef02-49e8-8d1d-2007c626d4db /tidb/cdc/default/default/upstream/7362134550768622449 {"id":7362134550768622449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15295f1d-ef02-49e8-8d1d-2007c626d4db {"id":"15295f1d-ef02-49e8-8d1d-2007c626d4db","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24195af5 15295f1d-ef02-49e8-8d1d-2007c626d4db /tidb/cdc/default/default/upstream/7362134550768622449 {"id":7362134550768622449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15295f1d-ef02-49e8-8d1d-2007c626d4db {"id":"15295f1d-ef02-49e8-8d1d-2007c626d4db","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24195af5 15295f1d-ef02-49e8-8d1d-2007c626d4db /tidb/cdc/default/default/upstream/7362134550768622449 {"id":7362134550768622449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.13494.out cli changefeed create --start-ts=449349036241846273 '--sink-uri=kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: 0ad21aa7-2734-4726-a49f-80da1a8fcf93 Info: {"upstream_id":7362134550768622449,"namespace":"default","id":"0ad21aa7-2734-4726-a49f-80da1a8fcf93","sink_uri":"kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:22:24.830830946+08:00","start_ts":449349036241846273,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"snappy","claim_check_storage_uri":""}},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349036241846273,"checkpoint_ts":449349036241846273,"checkpoint_time":"2024-04-26 19:22:19.863"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... [2024/04/26 19:22:24.997 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:22:25.086 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [Fri Apr 26 19:22:24 CST 2024] <<<<<< START cdc server in kafka_column_selector_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.2994629948.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/04/26 19:22:24.544 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/04/26 19:22:24.872 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:22:24.977 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:22:25.130 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:22:25.132 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:22:25.209 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:22:25.211 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:22:25.409 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:22:25.474 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:22:25.386 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:22:25.592 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/04/26 19:22:25.630 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:22:25.632 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:22:25.705 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:22:25.706 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:22:25.862 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:22:25.928 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:22:25.996 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:22:26.089 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:22:26.094 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:22:26.169 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:22:26.174 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] + set +x [2024/04/26 19:22:26.322 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:22:26.424 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:22:26.365 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:22:26.507 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:22:26.677 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:22:26.680 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:22:26.791 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:22:26.793 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:22:26.809 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/04/26 19:22:26.952 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:22:27.104 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:22:27.486 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:22:27.488 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:22:27.666 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:22:27.656 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:22:27.660 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:22:28.002 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/04/26 19:22:27.892 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:22:28.061 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:22:27 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f32cbb3-9a9a-495b-85b8-f069eed612cb {"id":"4f32cbb3-9a9a-495b-85b8-f069eed612cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a242d3df3 4f32cbb3-9a9a-495b-85b8-f069eed612cb /tidb/cdc/default/default/upstream/7362134575732528097 {"id":7362134575732528097,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f32cbb3-9a9a-495b-85b8-f069eed612cb {"id":"4f32cbb3-9a9a-495b-85b8-f069eed612cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a242d3df3 4f32cbb3-9a9a-495b-85b8-f069eed612cb /tidb/cdc/default/default/upstream/7362134575732528097 {"id":7362134575732528097,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4f32cbb3-9a9a-495b-85b8-f069eed612cb {"id":"4f32cbb3-9a9a-495b-85b8-f069eed612cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a242d3df3 4f32cbb3-9a9a-495b-85b8-f069eed612cb /tidb/cdc/default/default/upstream/7362134575732528097 {"id":7362134575732528097,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.cli.30000.out cli changefeed create --start-ts=449349037498040321 '--sink-uri=kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: test Info: {"upstream_id":7362134575732528097,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true","create_time":"2024-04-26T19:22:28.226790481+08:00","start_ts":449349037498040321,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":[{"matcher":["test.*"],"columns":["*","!b"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349037498040321,"checkpoint_ts":449349037498040321,"checkpoint_time":"2024-04-26 19:22:24.655"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later [2024/04/26 19:22:28.361 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:22:28.364 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/04/26 19:22:28.613 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:22:28.615 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] table ddl_attributes.finish_mark not exists for 2-th check, retry later [2024/04/26 19:22:28.810 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] table test.finish_mark not exists for 1-th check, retry later [2024/04/26 19:22:28.969 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:22:29.296 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:22:29.298 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:22:29.088 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v3m25.518040007s"] [2024/04/26 19:22:29.326 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 3m25.756606112s"] [2024/04/26 19:22:29.423 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:22:29.426 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] + set +x [Fri Apr 26 19:22:29 CST 2024] <<<<<< START kafka consumer in kafka_column_selector_avro case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml schema registry uri found: http://127.0.0.1:8088 Starting build checksum checker... table test.finishmark not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 3-th check, retry later table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finishmark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully table mark.finish_mark_3 not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... table ddl_attributes.finish_mark not exists for 4-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir wait process cdc.test exit for 2-th time... [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:22:34 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_attributes.finish_mark not exists for 5-th check, retry later table test.finishmark exists [2024/04/26 19:22:34.185 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/04/26 19:22:34.187 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/04/26 19:22:34.188 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=10.22515ms] [2024/04/26 19:22:34.194 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/04/26 19:22:34.196 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/04/26 19:22:34.196 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=8.592324ms] [2024/04/26 19:22:34.196 +08:00] [INFO] [main.go:95] ["compare checksum passed"] table mark.finish_mark_3 not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... [2024/04/26 19:22:36.078 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] cdc.test: no process found wait process cdc.test exit for 2-th time... process cdc.test already exit [Fri Apr 26 19:22:36 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> table mark.finish_mark_3 not exists for 4-th check, retry later [2024/04/26 19:22:36.567 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:22:36.571 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:22:36.890 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:22:36.892 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:22:36.903 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:22:36.947 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] table ddl_attributes.finish_mark not exists for 6-th check, retry later [2024/04/26 19:22:37.291 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:22:37.295 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:22:37.649 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:22:37.652 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:22:37.654 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:22:37.657 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/04/26 19:22:38.068 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:22:38.071 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:22:38.386 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:22:38.387 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:22:38.389 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/04/26 19:22:38.389 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:22:38.768 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:22:38.776 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] table ddl_attributes.finish_mark exists check diff successfully [2024/04/26 19:22:39.165 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] table mark.finish_mark_3 not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... [2024/04/26 19:22:39.168 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:22:39.186 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:22:39.188 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:22:39.482 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:22:39.489 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] wait process cdc.test exit for 2-th time... [2024/04/26 19:22:39.783 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:22:39.785 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:22:39.788 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:22:39.790 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/04/26 19:22:40.022 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:22:40.024 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:22:40.261 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:22:40.264 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/04/26 19:22:40.264 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/04/26 19:22:40.266 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2024/04/26 19:22:40.474 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/04/26 19:22:40.478 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/04/26 19:22:40.681 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/04/26 19:22:40.683 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/04/26 19:22:40.692 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/04/26 19:22:40.694 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [Fri Apr 26 19:22:40 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> table mark.finish_mark_3 not exists for 6-th check, retry later [2024/04/26 19:22:41.011 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/04/26 19:22:41.063 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/04/26 19:22:38.479 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/04/26 19:22:38.482 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/04/26 19:22:38.482 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=9eedbf0c-43f3-4e6c-abda-458f3dafd31b] [2024/04/26 19:22:38.482 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/04/26 19:22:38.483 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/04/26 19:22:38.664 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=06545d08-253d-4d31-9238-c07b879f9f8b] [captureID=9eedbf0c-43f3-4e6c-abda-458f3dafd31b] [processorDetail="{\"table_ids\":[104]}"] [2024/04/26 19:22:38.864 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=06545d08-253d-4d31-9238-c07b879f9f8b] [captureID=b169abfd-a008-4a56-9991-9ce86630c87e] [processorDetail="{\"table_ids\":[]}"] [2024/04/26 19:22:39.063 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=06545d08-253d-4d31-9238-c07b879f9f8b] [captureID=ee63afb4-6e20-4162-ba19-aae55f656c44] [processorDetail="{\"table_ids\":[106]}"] [2024/04/26 19:22:39.063 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"9eedbf0c-43f3-4e6c-abda-458f3dafd31b\":[{\"ID\":104,\"Changefeed\":\"06545d08-253d-4d31-9238-c07b879f9f8b\"}],\"b169abfd-a008-4a56-9991-9ce86630c87e\":[],\"ee63afb4-6e20-4162-ba19-aae55f656c44\":[{\"ID\":106,\"Changefeed\":\"06545d08-253d-4d31-9238-c07b879f9f8b\"}]}"] [2024/04/26 19:22:39.064 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=06545d08-253d-4d31-9238-c07b879f9f8b&target-cp-id=ee63afb4-6e20-4162-ba19-aae55f656c44&table-id=104"] [2024/04/26 19:22:39.114 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=104] [2024/04/26 19:22:39.114 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=9eedbf0c-43f3-4e6c-abda-458f3dafd31b] [targetCapture=ee63afb4-6e20-4162-ba19-aae55f656c44] table move_table.check1 exists check diff successfully table mark.finish_mark_3 not exists for 7-th check, retry later [2024/04/26 19:22:43.424 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/04/26 19:22:43.428 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/04/26 19:22:43.428 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=9eedbf0c-43f3-4e6c-abda-458f3dafd31b] [2024/04/26 19:22:43.429 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/04/26 19:22:43.429 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/04/26 19:22:43.614 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=06545d08-253d-4d31-9238-c07b879f9f8b] [captureID=9eedbf0c-43f3-4e6c-abda-458f3dafd31b] [processorDetail="{\"table_ids\":[108]}"] [2024/04/26 19:22:43.814 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=06545d08-253d-4d31-9238-c07b879f9f8b] [captureID=b169abfd-a008-4a56-9991-9ce86630c87e] [processorDetail="{\"table_ids\":[]}"] [2024/04/26 19:22:44.013 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=06545d08-253d-4d31-9238-c07b879f9f8b] [captureID=ee63afb4-6e20-4162-ba19-aae55f656c44] [processorDetail="{\"table_ids\":[106]}"] [2024/04/26 19:22:44.013 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"9eedbf0c-43f3-4e6c-abda-458f3dafd31b\":[{\"ID\":108,\"Changefeed\":\"06545d08-253d-4d31-9238-c07b879f9f8b\"}],\"b169abfd-a008-4a56-9991-9ce86630c87e\":[],\"ee63afb4-6e20-4162-ba19-aae55f656c44\":[{\"ID\":106,\"Changefeed\":\"06545d08-253d-4d31-9238-c07b879f9f8b\"}]}"] [2024/04/26 19:22:44.013 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=06545d08-253d-4d31-9238-c07b879f9f8b&target-cp-id=b169abfd-a008-4a56-9991-9ce86630c87e&table-id=106"] [2024/04/26 19:22:44.064 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/04/26 19:22:44.064 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=ee63afb4-6e20-4162-ba19-aae55f656c44] [targetCapture=b169abfd-a008-4a56-9991-9ce86630c87e] check diff successfully table move_table.check2 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 8-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh: line 1: 13531 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table move_table.check2 exists check diff successfully table mark.finish_mark_3 not exists for 9-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2024/04/26 19:22:47.071 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/04/26 19:22:47.349 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/04/26 19:22:47.352 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/04/26 19:22:47.564 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/04/26 19:22:47.567 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/04/26 19:22:47.571 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/04/26 19:22:47.573 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/04/26 19:22:47.781 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/04/26 19:22:47.795 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/04/26 19:22:47.980 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/04/26 19:22:47.982 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/04/26 19:22:47.997 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/04/26 19:22:47.999 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:22:47 CST 2024] <<<<<< run test case move_table success! >>>>>> [2024/04/26 19:22:48.194 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/04/26 19:22:48.210 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/04/26 19:22:48.472 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/04/26 19:22:48.475 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/04/26 19:22:48.495 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/04/26 19:22:48.498 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] table mark.finish_mark_3 not exists for 10-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/open_protocol_handle_key_only Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... [2024/04/26 19:22:48.973 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/04/26 19:22:49.004 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/04/26 19:22:49.497 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/04/26 19:22:49.502 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/04/26 19:22:49.612 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/04/26 19:22:49.615 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/04/26 19:22:49.960 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/04/26 19:22:50.169 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/04/26 19:22:50.426 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/04/26 19:22:50.431 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/04/26 19:22:50.784 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/04/26 19:22:50.790 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] table mark.finish_mark_3 not exists for 11-th check, retry later [2024/04/26 19:22:51.087 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/04/26 19:22:51.646 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/04/26 19:22:51.669 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/04/26 19:22:51.676 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] table mark.finish_mark_3 not exists for 12-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:22:51 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:22:55 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> table mark.finish_mark_3 not exists for 13-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 14-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // node [Pipeline] } table mark.finish_mark_3 not exists for 15-th check, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6893ce640013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:14479, start at 2024-04-26 19:23:00.392178227 +0800 CST m=+5.919807819 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:00.421 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:00.426 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:00.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 16-th check, retry later [2024/04/26 19:23:00.585 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m8.402604565s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6893ce640013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:14479, start at 2024-04-26 19:23:00.392178227 +0800 CST m=+5.919807819 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:00.421 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:00.426 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:00.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6893ce600002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:14554, start at 2024-04-26 19:23:00.377216281 +0800 CST m=+5.817995756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:00.384 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:00.376 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:00.376 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure table mark.finish_mark_3 not exists for 17-th check, retry later Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 18-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.15825.out cli tso query --pd=http://127.0.0.1:2379 table mark.finish_mark exists check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + set +x + tso='449349048120639489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349048120639489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:23:06 CST 2024] <<<<<< START cdc server in open_protocol_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.1586215864.out server --log-file /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:23:07 CST 2024] <<<<<< run test case multi_source success! >>>>>> table mark.finish_mark_3 not exists for 19-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 20-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:09 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73191368-20e1-4908-80dc-ab167ff1cfae {"id":"73191368-20e1-4908-80dc-ab167ff1cfae","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24c6a7f9 73191368-20e1-4908-80dc-ab167ff1cfae /tidb/cdc/default/default/upstream/7362134744595843912 {"id":7362134744595843912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73191368-20e1-4908-80dc-ab167ff1cfae {"id":"73191368-20e1-4908-80dc-ab167ff1cfae","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24c6a7f9 73191368-20e1-4908-80dc-ab167ff1cfae /tidb/cdc/default/default/upstream/7362134744595843912 {"id":7362134744595843912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73191368-20e1-4908-80dc-ab167ff1cfae {"id":"73191368-20e1-4908-80dc-ab167ff1cfae","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24c6a7f9 73191368-20e1-4908-80dc-ab167ff1cfae /tidb/cdc/default/default/upstream/7362134744595843912 {"id":7362134744595843912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.15913.out cli changefeed create --start-ts=449349048120639489 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: 29bf442d-d786-4444-88d1-19fc57806845 Info: {"upstream_id":7362134744595843912,"namespace":"default","id":"29bf442d-d786-4444-88d1-19fc57806845","sink_uri":"kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:23:10.123531685+08:00","start_ts":449349048120639489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349048120639489,"checkpoint_ts":449349048120639489,"checkpoint_time":"2024-04-26 19:23:05.177"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 21-th check, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 22-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689491ac0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:26761, start at 2024-04-26 19:23:12.899229341 +0800 CST m=+5.432233380 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:12.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:12.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:12.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3 -s + info='{ "upstream_id": 7362134333186869519, "namespace": "default", "id": "3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3", "state": "finished", "checkpoint_tso": 449349047359373315, "checkpoint_time": "2024-04-26 19:23:02.273", "error": null }' + echo '{ "upstream_id": 7362134333186869519, "namespace": "default", "id": "3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3", "state": "finished", "checkpoint_tso": 449349047359373315, "checkpoint_time": "2024-04-26 19:23:02.273", "error": null }' { "upstream_id": 7362134333186869519, "namespace": "default", "id": "3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3", "state": "finished", "checkpoint_tso": 449349047359373315, "checkpoint_time": "2024-04-26 19:23:02.273", "error": null } ++ echo '{' '"upstream_id":' 7362134333186869519, '"namespace":' '"default",' '"id":' '"3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3",' '"state":' '"finished",' '"checkpoint_tso":' 449349047359373315, '"checkpoint_time":' '"2024-04-26' '19:23:02.273",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7362134333186869519, '"namespace":' '"default",' '"id":' '"3a70bfa6-d8f2-4f0c-8461-b5b2cb7c14b3",' '"state":' '"finished",' '"checkpoint_tso":' 449349047359373315, '"checkpoint_time":' '"2024-04-26' '19:23:02.273",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:23:08 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> table test.finish_mark not exists for 2-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2672 0 --:--:-- --:--:-- --:--:-- 2695 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:23:02.239","puller_resolved_ts":"2024-04-26 19:22:54.339","last_synced_ts":"2024-04-26 19:20:45.638","now_ts":"2024-04-26 19:23:03.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:23:02.239","puller_resolved_ts":"2024-04-26' '19:22:54.339","last_synced_ts":"2024-04-26' '19:20:45.638","now_ts":"2024-04-26' '19:23:03.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 14194 8.6 0.0 13641224 139804 ? Sl 19:20 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 14233 5.6 0.0 13640136 133616 ? Sl 19:20 0:09 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714130589.1142075,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002c10700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130589.1142757,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130589.1448157,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00102b180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714130589.1448789,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130589.2173064,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00230e700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130589.2173605,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:23:12.918586+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:12.920824+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:13.03839+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table mark.finish_mark_3 not exists for 23-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689491ac0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:26761, start at 2024-04-26 19:23:12.899229341 +0800 CST m=+5.432233380 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:12.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:12.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:12.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689493e40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:26842, start at 2024-04-26 19:23:13.022550633 +0800 CST m=+5.456561938 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:13.028 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:13.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:13.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 24-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:23:17 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> [Fri Apr 26 19:23:17 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.2805328055.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 25-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [2024/04/26 19:23:15.954 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m34.568655564s] [2024/04/26 19:23:15.956 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) {"level":"warn","ts":"2024-04-26T19:23:18.920208+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:18.92182+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:19.039554+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table mark.finish_mark_3 not exists for 26-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:20 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdab9d0a-3308-4b99-866b-38b478a09a5a {"id":"cdab9d0a-3308-4b99-866b-38b478a09a5a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24f96bf4 cdab9d0a-3308-4b99-866b-38b478a09a5a /tidb/cdc/default/default/upstream/7362134801312435194 {"id":7362134801312435194,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdab9d0a-3308-4b99-866b-38b478a09a5a {"id":"cdab9d0a-3308-4b99-866b-38b478a09a5a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24f96bf4 cdab9d0a-3308-4b99-866b-38b478a09a5a /tidb/cdc/default/default/upstream/7362134801312435194 {"id":7362134801312435194,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdab9d0a-3308-4b99-866b-38b478a09a5a {"id":"cdab9d0a-3308-4b99-866b-38b478a09a5a","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a24f96bf4 cdab9d0a-3308-4b99-866b-38b478a09a5a /tidb/cdc/default/default/upstream/7362134801312435194 {"id":7362134801312435194,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 771a89ad-1fee-498e-8013-fae112d700dc Info: {"upstream_id":7362134801312435194,"namespace":"default","id":"771a89ad-1fee-498e-8013-fae112d700dc","sink_uri":"kafka://127.0.0.1:9092/ticdc-autorandom-test-22725?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:23:21.108186675+08:00","start_ts":449349052248621062,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349052248621062,"checkpoint_ts":449349052248621062,"checkpoint_time":"2024-04-26 19:23:20.924"} [Fri Apr 26 19:23:21 CST 2024] <<<<<< START kafka consumer in autorandom case >>>>>> table autorandom_test.table_a not exists for 1-th check, retry later [2024/04/26 19:23:21.992 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table mark.finish_mark_3 not exists for 27-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... table autorandom_test.table_a not exists for 2-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:23:23 CST 2024] <<<<<< run test case cdc success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:23:24.921859+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:24.923211+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:25.04134+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table mark.finish_mark_3 not exists for 28-th check, retry later table autorandom_test.table_a exists check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:23:26 CST 2024] <<<<<< run test case autorandom success! >>>>>> table mark.finish_mark_3 not exists for 29-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 30-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh: line 1: 15944 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:23:30.92298+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:30.92446+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:31.04314+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table mark.finish_mark_3 not exists for 31-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/canal_json_claim_check Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:23:32.91096+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:23:32.911004+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:23:32.911813+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:23:32.911848+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:23:33.028716+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:23:33.028762+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table mark.finish_mark_3 not exists for 32-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6895d4480012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:6394, start at 2024-04-26 19:23:33.551080083 +0800 CST m=+5.715362184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:33.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:33.522 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:33.522 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 33-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6895d4480012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:6394, start at 2024-04-26 19:23:33.551080083 +0800 CST m=+5.715362184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:33.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:33.522 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:33.522 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6895d8e00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-vf099, pid:6448, start at 2024-04-26 19:23:33.856889929 +0800 CST m=+5.894447496 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:33.864 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:33.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:33.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:23:36.923624+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:36.925987+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:37.044103+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table mark.finish_mark_3 not exists for 34-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:23:38 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.77957797.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 35-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table mark.finish_mark_3 not exists for 36-th check, retry later < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:41 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/314bab48-f9a8-4b86-9adf-6793612ae2eb {"id":"314bab48-f9a8-4b86-9adf-6793612ae2eb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a254c9cf2 314bab48-f9a8-4b86-9adf-6793612ae2eb /tidb/cdc/default/default/upstream/7362134887424267912 {"id":7362134887424267912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/314bab48-f9a8-4b86-9adf-6793612ae2eb {"id":"314bab48-f9a8-4b86-9adf-6793612ae2eb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a254c9cf2 314bab48-f9a8-4b86-9adf-6793612ae2eb /tidb/cdc/default/default/upstream/7362134887424267912 {"id":7362134887424267912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/314bab48-f9a8-4b86-9adf-6793612ae2eb {"id":"314bab48-f9a8-4b86-9adf-6793612ae2eb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a254c9cf2 314bab48-f9a8-4b86-9adf-6793612ae2eb /tidb/cdc/default/default/upstream/7362134887424267912 {"id":7362134887424267912,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 264d0662-39fe-4c8c-88cc-151ab62b6f50 Info: {"upstream_id":7362134887424267912,"namespace":"default","id":"264d0662-39fe-4c8c-88cc-151ab62b6f50","sink_uri":"kafka://127.0.0.1:9092/ticdc-force_replicate_table-test-5466?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:23:41.659240177+08:00","start_ts":449349056796033025,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349056796033025,"checkpoint_ts":449349056796033025,"checkpoint_time":"2024-04-26 19:23:38.271"} [Fri Apr 26 19:23:41 CST 2024] <<<<<< START kafka consumer in force_replicate_table case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:23:42.924644+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:42.927077+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:43.04501+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table mark.finish_mark_3 not exists for 37-th check, retry later table force_replicate_table.t0 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":1714130624.1146944,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002c10700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130624.1147466,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714130624.1461477,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00102b180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714130624.146206,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714130624.2194207,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00230e700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714130624.2194774,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table mark.finish_mark_3 not exists for 38-th check, retry later table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689668c80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:16961, start at 2024-04-26 19:23:43.053813832 +0800 CST m=+5.586981522 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:43.062 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:43.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:43.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689668c80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:16961, start at 2024-04-26 19:23:43.053813832 +0800 CST m=+5.586981522 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:43.062 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:43.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:43.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689669740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:17038, start at 2024-04-26 19:23:43.119506102 +0800 CST m=+5.562613927 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:43.126 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:43.120 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:43.120 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release table mark.finish_mark_3 exists table mark.finish_mark exists table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.18266.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... check diff successfully wait process cdc.test exit for 1-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:23:48.925043+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000f5f880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:48.928588+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0006e4fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:23:49.046683+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000ef0e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + set +x + tso='449349059379724289 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349059379724289 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:23:49 CST 2024] <<<<<< START cdc server in canal_json_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.1829518297.out server --log-file /tmp/tidb_cdc_test/canal_json_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_claim_check/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:23:50 CST 2024] <<<<<< run test case default_value success! >>>>>> table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:23:52 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d3d8f42-5a66-4f69-8b3e-e648c6899b8b {"id":"7d3d8f42-5a66-4f69-8b3e-e648c6899b8b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2571f2f7 7d3d8f42-5a66-4f69-8b3e-e648c6899b8b /tidb/cdc/default/default/upstream/7362134929663577284 {"id":7362134929663577284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d3d8f42-5a66-4f69-8b3e-e648c6899b8b {"id":"7d3d8f42-5a66-4f69-8b3e-e648c6899b8b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2571f2f7 7d3d8f42-5a66-4f69-8b3e-e648c6899b8b /tidb/cdc/default/default/upstream/7362134929663577284 {"id":7362134929663577284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d3d8f42-5a66-4f69-8b3e-e648c6899b8b {"id":"7d3d8f42-5a66-4f69-8b3e-e648c6899b8b","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2571f2f7 7d3d8f42-5a66-4f69-8b3e-e648c6899b8b /tidb/cdc/default/default/upstream/7362134929663577284 {"id":7362134929663577284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.18342.out cli changefeed create --start-ts=449349059379724289 '--sink-uri=kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/conf/changefeed.toml Create changefeed successfully! ID: 33531215-06e9-4e77-84db-08518c1d5923 Info: {"upstream_id":7362134929663577284,"namespace":"default","id":"33531215-06e9-4e77-84db-08518c1d5923","sink_uri":"kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:23:52.983342853+08:00","start_ts":449349059379724289,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/canal-json-claim-check"}},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349059379724289,"checkpoint_ts":449349059379724289,"checkpoint_time":"2024-04-26 19:23:48.127"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896e45c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:29107, start at 2024-04-26 19:23:50.952583935 +0800 CST m=+5.390162824 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:50.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:50.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:50.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896e45c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:29107, start at 2024-04-26 19:23:50.952583935 +0800 CST m=+5.390162824 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:50.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:50.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:50.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6896e4fc0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-64b3n, pid:29189, start at 2024-04-26 19:23:51.021411132 +0800 CST m=+5.371007430 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:51.030 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:51.024 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:51.024 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster table force_replicate_table.t6 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689723340018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:23431, start at 2024-04-26 19:23:54.996980388 +0800 CST m=+5.198465762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:55.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:55.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:55.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689723340018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:23431, start at 2024-04-26 19:23:54.996980388 +0800 CST m=+5.198465762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:55.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:55.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:55.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6897257c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-62m7h, pid:23509, start at 2024-04-26 19:23:55.113817365 +0800 CST m=+5.207421738 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:25:55.122 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:23:55.103 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:13:55.103 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.30452.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 1-th check, retry later table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 + set +x + tso='449349061492080641 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349061492080641 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:23:57 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3048630488.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.24730.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark exists check diff successfully run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 + set +x + tso='449349062087409665 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349062087409665 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:24:00 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:00 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92f0a1d6-d146-4f6b-ac7d-b3409d3fc237 {"id":"92f0a1d6-d146-4f6b-ac7d-b3409d3fc237","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a259201f8 92f0a1d6-d146-4f6b-ac7d-b3409d3fc237 /tidb/cdc/default/default/upstream/7362134958731471289 {"id":7362134958731471289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92f0a1d6-d146-4f6b-ac7d-b3409d3fc237 {"id":"92f0a1d6-d146-4f6b-ac7d-b3409d3fc237","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a259201f8 92f0a1d6-d146-4f6b-ac7d-b3409d3fc237 /tidb/cdc/default/default/upstream/7362134958731471289 {"id":7362134958731471289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92f0a1d6-d146-4f6b-ac7d-b3409d3fc237 {"id":"92f0a1d6-d146-4f6b-ac7d-b3409d3fc237","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a259201f8 92f0a1d6-d146-4f6b-ac7d-b3409d3fc237 /tidb/cdc/default/default/upstream/7362134958731471289 {"id":7362134958731471289,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.30541.out cli changefeed create --start-ts=449349061492080641 '--sink-uri=kafka://127.0.0.1:9092/ticdc-generate-column-test-28071?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: bb3a1bc4-455e-40df-988c-5eb838af8e98 Info: {"upstream_id":7362134958731471289,"namespace":"default","id":"bb3a1bc4-455e-40df-988c-5eb838af8e98","sink_uri":"kafka://127.0.0.1:9092/ticdc-generate-column-test-28071?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:01.047706237+08:00","start_ts":449349061492080641,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349061492080641,"checkpoint_ts":449349061492080641,"checkpoint_time":"2024-04-26 19:23:56.185"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 [Fri Apr 26 19:24:01 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2477724779.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later + set +x [Fri Apr 26 19:24:02 CST 2024] <<<<<< START kafka consumer in generate_column case >>>>>> + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 table generate_column.t not exists for 1-th check, retry later run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:04 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523b59cd-b1a3-4f97-b24c-265515f426d6 {"id":"523b59cd-b1a3-4f97-b24c-265515f426d6","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25988916 523b59cd-b1a3-4f97-b24c-265515f426d6 /tidb/cdc/default/default/upstream/7362134975890614702 {"id":7362134975890614702,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523b59cd-b1a3-4f97-b24c-265515f426d6 {"id":"523b59cd-b1a3-4f97-b24c-265515f426d6","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25988916 523b59cd-b1a3-4f97-b24c-265515f426d6 /tidb/cdc/default/default/upstream/7362134975890614702 {"id":7362134975890614702,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523b59cd-b1a3-4f97-b24c-265515f426d6 {"id":"523b59cd-b1a3-4f97-b24c-265515f426d6","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25988916 523b59cd-b1a3-4f97-b24c-265515f426d6 /tidb/cdc/default/default/upstream/7362134975890614702 {"id":7362134975890614702,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.24823.out cli changefeed create --start-ts=449349062087409665 '--sink-uri=kafka://127.0.0.1:9092/ticdc-split-region-test-22449?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 3e7e2863-6c4f-4bb8-909d-24c1c0cefb3b Info: {"upstream_id":7362134975890614702,"namespace":"default","id":"3e7e2863-6c4f-4bb8-909d-24c1c0cefb3b","sink_uri":"kafka://127.0.0.1:9092/ticdc-split-region-test-22449?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:04.940667962+08:00","start_ts":449349062087409665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349062087409665,"checkpoint_ts":449349062087409665,"checkpoint_time":"2024-04-26 19:23:58.456"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:24:06 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> table generate_column.t exists table generate_column.t1 not exists for 1-th check, retry later + set +x [Fri Apr 26 19:24:06 CST 2024] <<<<<< START kafka consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... table generate_column.t1 exists check diff failed 1-th time, retry later table split_region.test1 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release check diff successfully wait process cdc.test exit for 1-th time... table split_region.test1 exists table split_region.test2 exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:24:11 CST 2024] <<<<<< run test case generate_column success! >>>>>> Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh: line 1: 18382 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/open_protocol_claim_check Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898781c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:17363, start at 2024-04-26 19:24:16.786789702 +0800 CST m=+5.667236144 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:16.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:16.775 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:16.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898781c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:17363, start at 2024-04-26 19:24:16.786789702 +0800 CST m=+5.667236144 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:16.794 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:16.775 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:16.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689879100004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:17427, start at 2024-04-26 19:24:16.841981046 +0800 CST m=+5.628808460 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:16.848 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:16.836 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:16.836 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18633.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449349068187238405 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349068187238405 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349068187238405 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:24:23 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1866218664.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898efe0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:15134, start at 2024-04-26 19:24:24.480004613 +0800 CST m=+12.542916256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:24.527 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:24.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:24.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898efe0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:15134, start at 2024-04-26 19:24:24.480004613 +0800 CST m=+12.542916256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:24.527 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:24.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:24.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6898f024000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:15164, start at 2024-04-26 19:24:24.478613193 +0800 CST m=+12.377885425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:24.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:24.505 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:24.505 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:26 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36bb71e7-ecdc-4eb4-9505-e29b16278f1e {"id":"36bb71e7-ecdc-4eb4-9505-e29b16278f1e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25f01af4 36bb71e7-ecdc-4eb4-9505-e29b16278f1e /tidb/cdc/default/default/upstream/7362135078169172601 {"id":7362135078169172601,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36bb71e7-ecdc-4eb4-9505-e29b16278f1e {"id":"36bb71e7-ecdc-4eb4-9505-e29b16278f1e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25f01af4 36bb71e7-ecdc-4eb4-9505-e29b16278f1e /tidb/cdc/default/default/upstream/7362135078169172601 {"id":7362135078169172601,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36bb71e7-ecdc-4eb4-9505-e29b16278f1e {"id":"36bb71e7-ecdc-4eb4-9505-e29b16278f1e","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25f01af4 36bb71e7-ecdc-4eb4-9505-e29b16278f1e /tidb/cdc/default/default/upstream/7362135078169172601 {"id":7362135078169172601,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349068187238405 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18708.out cli changefeed create --start-ts=449349068187238405 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362135078169172601,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:24:26.603805794+08:00","start_ts":449349068187238405,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349068187238405,"checkpoint_ts":449349068187238405,"checkpoint_time":"2024-04-26 19:24:21.725"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully wait process cdc.test exit for 1-th time... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899029c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:19393, start at 2024-04-26 19:24:25.68814996 +0800 CST m=+5.700493716 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:25.695 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:25.689 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:25.689 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899029c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:19393, start at 2024-04-26 19:24:25.68814996 +0800 CST m=+5.700493716 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:25.695 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:25.689 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:25.689 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c6899040c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:19477, start at 2024-04-26 19:24:25.765512387 +0800 CST m=+5.641214265 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:26:25.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:24:25.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:14:25.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:24:28 CST 2024] <<<<<< run test case split_region success! >>>>>> Logging trace to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.16378.out cli tso query --pd=http://127.0.0.1:2379 table test.t1 exists + sleep 5 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.20704.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349070170357761 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349070170357761 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:24:32 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.1641016412.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449349070523990021 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349070523990021 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 16793 16.8 0.4 3682112 1629860 ? Sl 19:24 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 16794 24.1 0.4 3722556 1683428 ? Sl 19:24 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 16795 16.5 0.4 3682620 1615660 ? Sl 19:24 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 16797 23.0 0.4 3711292 1673076 ? Sl 19:24 0:06 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2513 0 --:--:-- --:--:-- --:--:-- 2505 100 243 100 243 0 0 2510 0 --:--:-- --:--:-- --:--:-- 2505 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:24:34.225","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:24:28.075","now_ts":"2024-04-26 19:24:35.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:24:34.225","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:24:28.075","now_ts":"2024-04-26' '19:24:35.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:24:34.225","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:24:28.075","now_ts":"2024-04-26' '19:24:35.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:35 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1de91c2-129c-44e6-b826-228f2901b7f0 {"id":"d1de91c2-129c-44e6-b826-228f2901b7f0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25f8ab1e d1de91c2-129c-44e6-b826-228f2901b7f0 /tidb/cdc/default/default/upstream/7362135077861397730 {"id":7362135077861397730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1de91c2-129c-44e6-b826-228f2901b7f0 {"id":"d1de91c2-129c-44e6-b826-228f2901b7f0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25f8ab1e d1de91c2-129c-44e6-b826-228f2901b7f0 /tidb/cdc/default/default/upstream/7362135077861397730 {"id":7362135077861397730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d1de91c2-129c-44e6-b826-228f2901b7f0 {"id":"d1de91c2-129c-44e6-b826-228f2901b7f0","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a25f8ab1e d1de91c2-129c-44e6-b826-228f2901b7f0 /tidb/cdc/default/default/upstream/7362135077861397730 {"id":7362135077861397730,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.16448.out cli changefeed create --start-ts=449349070170357761 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-simple-test-24160?protocol=open-protocol&partition-num=4&kafka-client-id=cdc_test_simple&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 17ef0fda-4e11-4cca-b06e-ad588bba81e0 Info: {"upstream_id":7362135077861397730,"namespace":"default","id":"17ef0fda-4e11-4cca-b06e-ad588bba81e0","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-simple-test-24160?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=cdc_test_simple\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:24:36.096238654+08:00","start_ts":449349070170357761,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349070170357761,"checkpoint_ts":449349070170357761,"checkpoint_time":"2024-04-26 19:24:29.290"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.20746.out cli tso query --pd=http://127.0.0.1:2379 + set +x [Fri Apr 26 19:24:37 CST 2024] <<<<<< START kafka consumer in simple case >>>>>> succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later + set +x + tso='449349072293724167 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349072293724167 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 26 19:24:38 CST 2024] <<<<<< START cdc server in open_protocol_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.2077720779.out server --log-file /tmp/tidb_cdc_test/open_protocol_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_claim_check/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:24:41 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb {"id":"40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2616a33c 40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb /tidb/cdc/default/default/upstream/7362135111882892236 {"id":7362135111882892236,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb {"id":"40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2616a33c 40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb /tidb/cdc/default/default/upstream/7362135111882892236 {"id":7362135111882892236,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb {"id":"40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2616a33c 40860b4e-2d84-4bcf-a6d9-7ad4173bb8cb /tidb/cdc/default/default/upstream/7362135111882892236 {"id":7362135111882892236,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.20824.out cli changefeed create --start-ts=449349070523990021 --target-ts=449349072293724167 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/conf/changefeed.toml Create changefeed successfully! ID: a3b9325d-7f7b-4be4-87ed-406d5fd3ba42 Info: {"upstream_id":7362135111882892236,"namespace":"default","id":"a3b9325d-7f7b-4be4-87ed-406d5fd3ba42","sink_uri":"kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-04-26T19:24:42.359746855+08:00","start_ts":449349070523990021,"target_ts":449349072293724167,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"lz4","claim_check_storage_uri":"file:///tmp/open-protocol-claim-check"}},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349070523990021,"checkpoint_ts":449349070523990021,"checkpoint_time":"2024-04-26 19:24:30.639"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... check data successfully wait process cdc.test exit for 1-th time... + set +x table test.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:24:44 CST 2024] <<<<<< run test case simple success! >>>>>> table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:24:49 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh: line 1: 20856 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:25:00 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri Apr 26 19:25:03 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D0CC4C276E90 < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:25:09 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689c7ee00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:17402, start at 2024-04-26 19:25:22.753685069 +0800 CST m=+15.924165865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:22.830 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:22.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:22.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689c7ee00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:17402, start at 2024-04-26 19:25:22.753685069 +0800 CST m=+15.924165865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:22.830 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:22.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:22.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689c8c7c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:22121, start at 2024-04-26 19:25:23.661025863 +0800 CST m=+5.537729423 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:23.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:23.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:23.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689c88ac0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:17441, start at 2024-04-26 19:25:23.389945446 +0800 CST m=+16.380758900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:23.513 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:23.576 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:23.576 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689c8c7c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:22121, start at 2024-04-26 19:25:23.661025863 +0800 CST m=+5.537729423 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:23.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:23.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:23.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c689c8d340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-81rwg, pid:22208, start at 2024-04-26 19:25:23.702969515 +0800 CST m=+5.471393910 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:27:23.709 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:25:23.711 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:15:23.711 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.18674.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 26 19:25:28 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.2347523477.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449349085770022913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349085770022913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Fri Apr 26 19:25:30 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.1869418696.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:31 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7735d214-eb76-42aa-9d3f-bc06419e3fbf {"id":"7735d214-eb76-42aa-9d3f-bc06419e3fbf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26f7bef5 7735d214-eb76-42aa-9d3f-bc06419e3fbf /tidb/cdc/default/default/upstream/7362135363751746394 {"id":7362135363751746394,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7735d214-eb76-42aa-9d3f-bc06419e3fbf {"id":"7735d214-eb76-42aa-9d3f-bc06419e3fbf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26f7bef5 7735d214-eb76-42aa-9d3f-bc06419e3fbf /tidb/cdc/default/default/upstream/7362135363751746394 {"id":7362135363751746394,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7735d214-eb76-42aa-9d3f-bc06419e3fbf {"id":"7735d214-eb76-42aa-9d3f-bc06419e3fbf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26f7bef5 7735d214-eb76-42aa-9d3f-bc06419e3fbf /tidb/cdc/default/default/upstream/7362135363751746394 {"id":7362135363751746394,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7362135363751746394,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-4827?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:25:31.893878486+08:00","start_ts":449349085709205505,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349085709205505,"checkpoint_ts":449349085709205505,"checkpoint_time":"2024-04-26 19:25:28.566"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7362135363751746394,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-32748?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:25:32.080736445+08:00","start_ts":449349085709205505,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349085709205505,"checkpoint_ts":449349085709205505,"checkpoint_time":"2024-04-26 19:25:28.566"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7362135363751746394,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-29429?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:25:32.28206946+08:00","start_ts":449349085709205505,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349085709205505,"checkpoint_ts":449349085709205505,"checkpoint_time":"2024-04-26 19:25:28.566"} [Fri Apr 26 19:25:32 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Fri Apr 26 19:25:32 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Fri Apr 26 19:25:32 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:25:34 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00f5845e-9f44-49ca-8cb9-cd0add88f872 {"id":"00f5845e-9f44-49ca-8cb9-cd0add88f872","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26cd1013 00f5845e-9f44-49ca-8cb9-cd0add88f872 /tidb/cdc/default/default/upstream/7362135312768691584 {"id":7362135312768691584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00f5845e-9f44-49ca-8cb9-cd0add88f872 {"id":"00f5845e-9f44-49ca-8cb9-cd0add88f872","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26cd1013 00f5845e-9f44-49ca-8cb9-cd0add88f872 /tidb/cdc/default/default/upstream/7362135312768691584 {"id":7362135312768691584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00f5845e-9f44-49ca-8cb9-cd0add88f872 {"id":"00f5845e-9f44-49ca-8cb9-cd0add88f872","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a26cd1013 00f5845e-9f44-49ca-8cb9-cd0add88f872 /tidb/cdc/default/default/upstream/7362135312768691584 {"id":7362135312768691584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449349087530844170, "checkpoint_time": "2024-04-26 19:25:35.515", "error": null }' + echo '{ "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449349087530844170, "checkpoint_time": "2024-04-26 19:25:35.515", "error": null }' { "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449349087530844170, "checkpoint_time": "2024-04-26 19:25:35.515", "error": null } ++ echo '{' '"upstream_id":' 7362135363751746394, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449349087530844170, '"checkpoint_time":' '"2024-04-26' '19:25:35.515",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135363751746394, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449349087530844170, '"checkpoint_time":' '"2024-04-26' '19:25:35.515",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449349090598191110, "checkpoint_time": "2024-04-26 19:25:47.216", "error": null }' + echo '{ "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449349090598191110, "checkpoint_time": "2024-04-26 19:25:47.216", "error": null }' { "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449349090598191110, "checkpoint_time": "2024-04-26 19:25:47.216", "error": null } ++ echo '{' '"upstream_id":' 7362135363751746394, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449349090598191110, '"checkpoint_time":' '"2024-04-26' '19:25:47.216",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7362135363751746394, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449349090598191110, '"checkpoint_time":' '"2024-04-26' '19:25:47.216",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449349087019925544, "checkpoint_time": "2024-04-26 19:25:33.566", "error": { "time": "2024-04-26T19:25:36.277237697+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:25:35.416 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449349087504891911. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449349087504891911]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449349087019925544, "checkpoint_time": "2024-04-26 19:25:33.566", "error": { "time": "2024-04-26T19:25:36.277237697+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:25:35.416 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449349087504891911. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449349087504891911]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7362135363751746394, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449349087019925544, "checkpoint_time": "2024-04-26 19:25:33.566", "error": { "time": "2024-04-26T19:25:36.277237697+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrHandleDDLFailed", "message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:25:35.416 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449349087504891911. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449349087504891911]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7362135363751746394, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449349087019925544, '"checkpoint_time":' '"2024-04-26' '19:25:33.566",' '"error":' '{' '"time":' '"2024-04-26T19:25:36.277237697+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:25:35.416 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449349087504891911. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449349087504891911]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7362135363751746394, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449349087019925544, '"checkpoint_time":' '"2024-04-26' '19:25:33.566",' '"error":' '{' '"time":' '"2024-04-26T19:25:36.277237697+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:25:35.416 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449349087504891911. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449349087504891911]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:25:35.416 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449349087504891911. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449349087504891911]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-04-26 19:25:35.416 +0800 CST, Err:, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449349087504891911. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449349087504891911]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 26 19:25:50 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Fri Apr 26 19:25:54 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ true != \n\o ]] + set +x + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.1876118763.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Fri Apr 26 19:26:14 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 6734 0 --:--:-- --:--:-- --:--:-- 6757 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:24:34.225","puller_resolved_ts":"2024-04-26 19:24:34.225","last_synced_ts":"2024-04-26 19:24:28.075","now_ts":"2024-04-26 19:26:45.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:24:34.225","puller_resolved_ts":"2024-04-26' '19:24:34.225","last_synced_ts":"2024-04-26' '19:24:28.075","now_ts":"2024-04-26' '19:26:45.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:24:34.225","puller_resolved_ts":"2024-04-26' '19:24:34.225","last_synced_ts":"2024-04-26' '19:24:28.075","now_ts":"2024-04-26' '19:26:45.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a258000001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:19598, start at 2024-04-26 19:26:58.571217805 +0800 CST m=+20.684558930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:58.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:58.658 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:58.658 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a258000001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:19598, start at 2024-04-26 19:26:58.571217805 +0800 CST m=+20.684558930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:58.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:58.658 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:58.658 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2577c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-m5jfg, pid:19622, start at 2024-04-26 19:26:58.566544348 +0800 CST m=+20.486439008 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:28:58.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:26:58.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:16:58.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri Apr 26 19:27:03 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2081920821.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:06 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bb3335b-c89f-418d-bc5e-200673cd1eaf {"id":"1bb3335b-c89f-418d-bc5e-200673cd1eaf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a282ca81c 1bb3335b-c89f-418d-bc5e-200673cd1eaf /tidb/cdc/default/default/upstream/7362135703403891768 {"id":7362135703403891768,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bb3335b-c89f-418d-bc5e-200673cd1eaf {"id":"1bb3335b-c89f-418d-bc5e-200673cd1eaf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a282ca81c 1bb3335b-c89f-418d-bc5e-200673cd1eaf /tidb/cdc/default/default/upstream/7362135703403891768 {"id":7362135703403891768,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bb3335b-c89f-418d-bc5e-200673cd1eaf {"id":"1bb3335b-c89f-418d-bc5e-200673cd1eaf","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a282ca81c 1bb3335b-c89f-418d-bc5e-200673cd1eaf /tidb/cdc/default/default/upstream/7362135703403891768 {"id":7362135703403891768,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.20853.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-event-filter-4184?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: db324e03-23b5-417f-a900-796ec388ae32 Info: {"upstream_id":7362135703403891768,"namespace":"default","id":"db324e03-23b5-417f-a900-796ec388ae32","sink_uri":"kafka://127.0.0.1:9092/ticdc-event-filter-4184?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-04-26T19:27:07.038572799+08:00","start_ts":449349111449649155,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349111449649155,"checkpoint_ts":449349111449649155,"checkpoint_time":"2024-04-26 19:27:06.758"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 26 19:27:08 CST 2024] <<<<<< START kafka consumer in event_filter case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2f4140013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:19739, start at 2024-04-26 19:27:08.588537293 +0800 CST m=+5.685902765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:08.595 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:08.599 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:08.599 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2f4140013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:19739, start at 2024-04-26 19:27:08.588537293 +0800 CST m=+5.685902765 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:08.595 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:08.599 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:08.599 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68a2f5640003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:19816, start at 2024-04-26 19:27:08.633919306 +0800 CST m=+5.632945833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:29:08.641 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:27:08.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:17:08.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table event_filter.t1 does not exists table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21098.out cli tso query --pd=http://127.0.0.1:2379 table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later + set +x + tso='449349113242976257 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349113242976257 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349113242976257 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:27:15 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2113321135.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:27:18 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48082031-c205-493a-86ee-cbb0b33a9bdb {"id":"48082031-c205-493a-86ee-cbb0b33a9bdb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28910af6 48082031-c205-493a-86ee-cbb0b33a9bdb /tidb/cdc/default/default/upstream/7362135811643287128 {"id":7362135811643287128,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48082031-c205-493a-86ee-cbb0b33a9bdb {"id":"48082031-c205-493a-86ee-cbb0b33a9bdb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28910af6 48082031-c205-493a-86ee-cbb0b33a9bdb /tidb/cdc/default/default/upstream/7362135811643287128 {"id":7362135811643287128,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/48082031-c205-493a-86ee-cbb0b33a9bdb {"id":"48082031-c205-493a-86ee-cbb0b33a9bdb","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a28910af6 48082031-c205-493a-86ee-cbb0b33a9bdb /tidb/cdc/default/default/upstream/7362135811643287128 {"id":7362135811643287128,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349113242976257 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21191.out cli changefeed create --start-ts=449349113242976257 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362135811643287128,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:27:18.524404146+08:00","start_ts":449349113242976257,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349113242976257,"checkpoint_ts":449349113242976257,"checkpoint_time":"2024-04-26 19:27:13.599"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table event_filter.t_alter exists + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table event_filter.finish_mark exists check diff failed 1-th time, retry later table test.t1 exists + sleep 5 check diff failed 2-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 26 19:27:26 CST 2024] <<<<<< run test case event_filter success! >>>>>> + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 19739 10.7 0.0 2731760 218096 ? Sl 19:27 0:02 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714130822892400871.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 19743 3.2 0.0 2212916 160100 ? Sl 19:27 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714130822899300891.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 19816 11.4 0.0 3157760 207248 ? Sl 19:27 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714130822994953113.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1547 0 --:--:-- --:--:-- --:--:-- 1547 100 243 100 243 0 0 1547 0 --:--:-- --:--:-- --:--:-- 1547 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:27:25.999","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:27:20.000","now_ts":"2024-04-26 19:27:27.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:27:25.999","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:27:20.000","now_ts":"2024-04-26' '19:27:27.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:27:25.999","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:27:20.000","now_ts":"2024-04-26' '19:27:27.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2686 0 --:--:-- --:--:-- --:--:-- 2695 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:29:36.150","puller_resolved_ts":"2024-04-26 19:29:29.100","last_synced_ts":"2024-04-26 19:27:20.000","now_ts":"2024-04-26 19:29:37.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:29:36.150","puller_resolved_ts":"2024-04-26' '19:29:29.100","last_synced_ts":"2024-04-26' '19:27:20.000","now_ts":"2024-04-26' '19:29:37.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:29:36.150","puller_resolved_ts":"2024-04-26' '19:29:29.100","last_synced_ts":"2024-04-26' '19:27:20.000","now_ts":"2024-04-26' '19:29:37.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ad7570000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:22272, start at 2024-04-26 19:30:00.683894845 +0800 CST m=+5.427901929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:00.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:00.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:00.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ad7570000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:22272, start at 2024-04-26 19:30:00.683894845 +0800 CST m=+5.427901929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:00.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:00.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:00.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68ad761c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:22362, start at 2024-04-26 19:30:00.746440624 +0800 CST m=+5.356087397 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:00.753 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:00.711 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:00.711 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.23581.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349158415630337 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349158415630337 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349158415630337 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:30:07 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2361923621.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:30:10 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb688836-376a-4fa8-8b67-76d0b77747a5 {"id":"cb688836-376a-4fa8-8b67-76d0b77747a5","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b3038f7 cb688836-376a-4fa8-8b67-76d0b77747a5 /tidb/cdc/default/default/upstream/7362136539199872765 {"id":7362136539199872765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb688836-376a-4fa8-8b67-76d0b77747a5 {"id":"cb688836-376a-4fa8-8b67-76d0b77747a5","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b3038f7 cb688836-376a-4fa8-8b67-76d0b77747a5 /tidb/cdc/default/default/upstream/7362136539199872765 {"id":7362136539199872765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb688836-376a-4fa8-8b67-76d0b77747a5 {"id":"cb688836-376a-4fa8-8b67-76d0b77747a5","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2b3038f7 cb688836-376a-4fa8-8b67-76d0b77747a5 /tidb/cdc/default/default/upstream/7362136539199872765 {"id":7362136539199872765,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349158415630337 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.23667.out cli changefeed create --start-ts=449349158415630337 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362136539199872765,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:30:10.788612479+08:00","start_ts":449349158415630337,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349158415630337,"checkpoint_ts":449349158415630337,"checkpoint_time":"2024-04-26 19:30:05.919"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 10348 0 --:--:-- --:--:-- --:--:-- 10478 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:30:05.919","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:30:32.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:30:05.919","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:30:32.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:30:05.919","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:30:32.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b0ea64000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:24581, start at 2024-04-26 19:30:57.317435189 +0800 CST m=+5.378528202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:57.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:57.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:57.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b0ea64000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:24581, start at 2024-04-26 19:30:57.317435189 +0800 CST m=+5.378528202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:57.323 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:57.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:57.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68b0ecb00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:24669, start at 2024-04-26 19:30:57.478204243 +0800 CST m=+5.394531730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:32:57.485 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:30:57.452 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:20:57.452 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25928.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349173275525121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349173275525121 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349173275525121 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:31:04 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2596125963.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:31:07 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4334df33-0497-4391-b176-ecbb364bd271 {"id":"4334df33-0497-4391-b176-ecbb364bd271","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c0dd9f7 4334df33-0497-4391-b176-ecbb364bd271 /tidb/cdc/default/default/upstream/7362136793599499181 {"id":7362136793599499181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4334df33-0497-4391-b176-ecbb364bd271 {"id":"4334df33-0497-4391-b176-ecbb364bd271","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c0dd9f7 4334df33-0497-4391-b176-ecbb364bd271 /tidb/cdc/default/default/upstream/7362136793599499181 {"id":7362136793599499181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4334df33-0497-4391-b176-ecbb364bd271 {"id":"4334df33-0497-4391-b176-ecbb364bd271","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2c0dd9f7 4334df33-0497-4391-b176-ecbb364bd271 /tidb/cdc/default/default/upstream/7362136793599499181 {"id":7362136793599499181,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349173275525121 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26007.out cli changefeed create --start-ts=449349173275525121 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362136793599499181,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:31:07.459471061+08:00","start_ts":449349173275525121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349173275525121,"checkpoint_ts":449349173275525121,"checkpoint_time":"2024-04-26 19:31:02.605"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2677 0 --:--:-- --:--:-- --:--:-- 2695 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:31:02.605","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:31:08.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:31:02.605","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:31:08.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:31:02.605","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:31:08.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-04-26 19:31:02.605' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:31:02.605","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:31:08.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:31:02.605","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:31:08.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-04-26 19:31:09' + echo 'sink_checkpoint_ts is 2024-04-26' 19:31:02.605 sink_checkpoint_ts is 2024-04-26 19:31:02.605 ++ date -d '2024-04-26 19:31:02.605' +%s + checkpoint_timestamp=1714131062 ++ date -d '2024-04-26 19:31:09' +%s + current_timestamp=1714131069 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2245 0 --:--:-- --:--:-- --:--:-- 2229 100 243 100 243 0 0 2243 0 --:--:-- --:--:-- --:--:-- 2229 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:31:15.106","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:31:09.106","now_ts":"2024-04-26 19:31:16.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:31:15.106","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:31:09.106","now_ts":"2024-04-26' '19:31:16.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:31:15.106","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:31:09.106","now_ts":"2024-04-26' '19:31:16.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 6171 0 --:--:-- --:--:-- --:--:-- 6314 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:33:25.205","puller_resolved_ts":"2024-04-26 19:33:19.205","last_synced_ts":"2024-04-26 19:31:09.106","now_ts":"2024-04-26 19:33:26.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:33:25.205","puller_resolved_ts":"2024-04-26' '19:33:19.205","last_synced_ts":"2024-04-26' '19:31:09.106","now_ts":"2024-04-26' '19:33:26.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 23941 7.2 0.0 13715468 141840 ? Sl 19:30 0:11 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 24010 5.0 0.0 13453004 134384 ? Sl 19:30 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714131212.3469186,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002f748c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714131212.34699,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714131212.5089705,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0032aee00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714131212.5090344,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714131213.2518332,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0026adc00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714131213.2518952,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:33:37.024325+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:37.101291+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:37.198162+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:43.025779+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:43.102829+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:43.199667+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:33:49.026188+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:49.10328+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:49.201532+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:33:55.027963+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:55.104005+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:33:55.202172+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:33:57.016666+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:33:57.016711+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:33:57.088615+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:33:57.08865+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-04-26T19:33:57.188688+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-04-26T19:33:57.188728+0800","logger":"etcd-client","caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:34:01.028978+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:34:01.105058+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:34:01.203694+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:34:07.030267+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:34:07.106573+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:34:07.2047+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714131247.3488624,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002f748c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714131247.348898,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714131247.5106237,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0032aee00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714131247.5106633,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714131248.253494,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0026adc00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714131248.2535357,"caller":"v3@v3.5.10/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-04-26T19:34:13.03189+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e961c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:34:13.107455+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010101c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-04-26T19:34:13.206051+0800","logger":"etcd-client","caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000fae540/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68be7be40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:27168, start at 2024-04-26 19:34:39.617557773 +0800 CST m=+5.409651416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:36:39.624 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:34:39.609 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:24:39.609 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68be7be40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:27168, start at 2024-04-26 19:34:39.617557773 +0800 CST m=+5.409651416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:36:39.624 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:34:39.609 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:24:39.609 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68be7b08000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:27247, start at 2024-04-26 19:34:39.578826842 +0800 CST m=+5.278647762 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:36:39.585 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:34:39.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:24:39.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.28477.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349231524708353 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349231524708353 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349231524708353 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:34:46 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2851028512.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:34:49 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aed67433-52d8-4acc-8a5b-1681ee22ae24 {"id":"aed67433-52d8-4acc-8a5b-1681ee22ae24","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2f7208f5 aed67433-52d8-4acc-8a5b-1681ee22ae24 /tidb/cdc/default/default/upstream/7362137752455297473 {"id":7362137752455297473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aed67433-52d8-4acc-8a5b-1681ee22ae24 {"id":"aed67433-52d8-4acc-8a5b-1681ee22ae24","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2f7208f5 aed67433-52d8-4acc-8a5b-1681ee22ae24 /tidb/cdc/default/default/upstream/7362137752455297473 {"id":7362137752455297473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aed67433-52d8-4acc-8a5b-1681ee22ae24 {"id":"aed67433-52d8-4acc-8a5b-1681ee22ae24","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a2f7208f5 aed67433-52d8-4acc-8a5b-1681ee22ae24 /tidb/cdc/default/default/upstream/7362137752455297473 {"id":7362137752455297473,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349231524708353 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.28560.out cli changefeed create --start-ts=449349231524708353 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362137752455297473,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:34:49.71204059+08:00","start_ts":449349231524708353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349231524708353,"checkpoint_ts":449349231524708353,"checkpoint_time":"2024-04-26 19:34:44.808"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 26594 15.8 0.4 3681596 1615776 ? Sl 19:34 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 26595 21.4 0.4 3724600 1682236 ? Sl 19:34 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 26596 15.6 0.4 3666748 1614640 ? Sl 19:34 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 26598 20.4 0.4 3705152 1673592 ? Sl 19:34 0:05 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3537 0 --:--:-- --:--:-- --:--:-- 3573 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:34:57.308","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:34:51.609","now_ts":"2024-04-26 19:34:58.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:34:57.308","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:34:51.609","now_ts":"2024-04-26' '19:34:58.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:34:57.308","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:34:51.609","now_ts":"2024-04-26' '19:34:58.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 6913 0 --:--:-- --:--:-- --:--:-- 6951 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:34:58.309","puller_resolved_ts":"2024-04-26 19:34:58.309","last_synced_ts":"2024-04-26 19:34:51.609","now_ts":"2024-04-26 19:37:08.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:34:58.309","puller_resolved_ts":"2024-04-26' '19:34:58.309","last_synced_ts":"2024-04-26' '19:34:51.609","now_ts":"2024-04-26' '19:37:08.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:34:58.309","puller_resolved_ts":"2024-04-26' '19:34:58.309","last_synced_ts":"2024-04-26' '19:34:51.609","now_ts":"2024-04-26' '19:37:08.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68c8faf00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:29647, start at 2024-04-26 19:37:31.61939866 +0800 CST m=+5.399314116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:39:31.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:37:31.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:27:31.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68c8faf00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:29647, start at 2024-04-26 19:37:31.61939866 +0800 CST m=+5.399314116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:39:31.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:37:31.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:27:31.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68c8fdf80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:29700, start at 2024-04-26 19:37:31.779242365 +0800 CST m=+5.466580977 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:39:31.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:37:31.774 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:27:31.774 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.30896.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349276619243521 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349276619243521 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349276619243521 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:37:38 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3092430926.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:37:41 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38b409b3-7584-4c14-9f8a-9edfd16f6bde {"id":"38b409b3-7584-4c14-9f8a-9edfd16f6bde","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a3213bdf8 38b409b3-7584-4c14-9f8a-9edfd16f6bde /tidb/cdc/default/default/upstream/7362138481006862887 {"id":7362138481006862887,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38b409b3-7584-4c14-9f8a-9edfd16f6bde {"id":"38b409b3-7584-4c14-9f8a-9edfd16f6bde","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a3213bdf8 38b409b3-7584-4c14-9f8a-9edfd16f6bde /tidb/cdc/default/default/upstream/7362138481006862887 {"id":7362138481006862887,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38b409b3-7584-4c14-9f8a-9edfd16f6bde {"id":"38b409b3-7584-4c14-9f8a-9edfd16f6bde","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a3213bdf8 38b409b3-7584-4c14-9f8a-9edfd16f6bde /tidb/cdc/default/default/upstream/7362138481006862887 {"id":7362138481006862887,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349276619243521 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.30976.out cli changefeed create --start-ts=449349276619243521 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362138481006862887,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:37:41.738398625+08:00","start_ts":449349276619243521,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349276619243521,"checkpoint_ts":449349276619243521,"checkpoint_time":"2024-04-26 19:37:36.830"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 29643 3.6 0.0 1966364 162932 ? Sl 19:37 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714131446211749715.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 29647 10.9 0.0 2196940 204024 ? Sl 19:37 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714131446214554821.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 29700 11.4 0.0 2115268 203548 ? Sl 19:37 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714131446306245718.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3702 0 --:--:-- --:--:-- --:--:-- 3738 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:37:47.280","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-26 19:37:43.630","now_ts":"2024-04-26 19:37:48.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:37:47.280","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:37:43.630","now_ts":"2024-04-26' '19:37:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:37:47.280","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-26' '19:37:43.630","now_ts":"2024-04-26' '19:37:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2646 0 --:--:-- --:--:-- --:--:-- 2662 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-26 19:39:57.431","puller_resolved_ts":"2024-04-26 19:39:52.430","last_synced_ts":"2024-04-26 19:37:43.630","now_ts":"2024-04-26 19:39:58.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:39:57.431","puller_resolved_ts":"2024-04-26' '19:39:52.430","last_synced_ts":"2024-04-26' '19:37:43.630","now_ts":"2024-04-26' '19:39:58.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-26' '19:39:57.431","puller_resolved_ts":"2024-04-26' '19:39:52.430","last_synced_ts":"2024-04-26' '19:37:43.630","now_ts":"2024-04-26' '19:39:58.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Starting Downstream PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release Starting Upstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68d33fe40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:31950, start at 2024-04-26 19:40:19.834453132 +0800 CST m=+5.508447551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:19.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:19.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:19.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68d33fe40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:31950, start at 2024-04-26 19:40:19.834453132 +0800 CST m=+5.508447551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:19.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:19.833 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:19.833 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63c68d341480010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-527-t065j, pid:32017, start at 2024-04-26 19:40:19.942031485 +0800 CST m=+5.527431916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240426-19:42:19.949 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240426-19:40:19.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240426-19:30:19.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v7.5.1-12-g9002cc34d Edition: Community Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314 Git Branch: HEAD UTC Build Time: 2024-04-18 07:24:48 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Raft Proxy Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981 Git Commit Branch: HEAD UTC Build Time: 2024-04-18 07:28:40 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.33302.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449349320686436355 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 449349320686436355 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449349320686436355 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 26 19:40:26 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3334233344.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) > GET /debug/info HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 26 Apr 2024 11:40:29 GMT < Content-Length: 613 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d467eeca-63ef-48f0-9bce-76e62553c0c6 {"id":"d467eeca-63ef-48f0-9bce-76e62553c0c6","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a34ac67f8 d467eeca-63ef-48f0-9bce-76e62553c0c6 /tidb/cdc/default/default/upstream/7362139210919887036 {"id":7362139210919887036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d467eeca-63ef-48f0-9bce-76e62553c0c6 {"id":"d467eeca-63ef-48f0-9bce-76e62553c0c6","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a34ac67f8 d467eeca-63ef-48f0-9bce-76e62553c0c6 /tidb/cdc/default/default/upstream/7362139210919887036 {"id":7362139210919887036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d467eeca-63ef-48f0-9bce-76e62553c0c6 {"id":"d467eeca-63ef-48f0-9bce-76e62553c0c6","address":"127.0.0.1:8300","version":"v7.5.1-57-g980f60a5b"} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f1a34ac67f8 d467eeca-63ef-48f0-9bce-76e62553c0c6 /tidb/cdc/default/default/upstream/7362139210919887036 {"id":7362139210919887036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449349320686436355 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.33386.out cli changefeed create --start-ts=449349320686436355 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7362139210919887036,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-26T19:40:29.788667259+08:00","start_ts":449349320686436355,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-57-g980f60a5b","resolved_ts":449349320686436355,"checkpoint_ts":449349320686436355,"checkpoint_time":"2024-04-26 19:40:24.933"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 9940 0 --:--:-- --:--:-- --:--:-- 10041 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-26 19:40:24.933","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-26 19:40:51.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:40:24.933","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:40:51.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-26' '19:40:24.933","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-26' '19:40:51.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Fri Apr 26 19:41:01 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Fri Apr 26 19:41:01 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-527/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS