Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-7.5/pull_dm_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k Agent pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k is provisioned from template pingcap_tiflow_release-7_5_pull_dm_integration_test_272-g9321-82g79 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "808afca36a59ea8c0887e1b1147d1d90eadef12a" jenkins/label: "pingcap_tiflow_release-7_5_pull_dm_integration_test_272-g9321" name: "pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git rev-list --no-walk cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=9b08d1f9-c539-4daf-97e9-e3063cb227f3 JENKINS_NODE_COOKIE=054f3b74-84b5-4bd1-94ef-9efc5bdc9120 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_dm_integration_test","buildid":"1783813079608332288","prowjobid":"9b08d1f9-c539-4daf-97e9-e3063cb227f3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"823a3899cc17bf14ad7875ab0cc69092186abc4a","base_link":"https://github.com/pingcap/tiflow/commit/823a3899cc17bf14ad7875ab0cc69092186abc4a","pulls":[{"number":10970,"author":"3AceShowHand","sha":"aa323aa99660421afc797ea115dbb30242e573cf","title":"*: release-7.5 bump tidb dependencies","link":"https://github.com/pingcap/tiflow/pull/10970","commit_link":"https://github.com/pingcap/tiflow/pull/10970/commits/aa323aa99660421afc797ea115dbb30242e573cf","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-7.5/pull_dm_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#272 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1783813079608332288 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_dm_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=cb9bc9e8822a01a5d59a2f670fb429e588065145 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-7_5_pull_dm_integration_test_272-g9321 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-24341cbb37398a3d76ef457a002bbb77ebf3752146c282247622af04dd7ac429 NODE_LABELS=pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k pingcap_tiflow_release-7_5_pull_dm_integration_test_272-g9321 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/display/redirect BUILD_NUMBER=272 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build1066160888=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k -c golang -- bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-7-5-pull-dm-integration-test-272-g-p9c5k -c golang -- bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 2143 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: d0ac9a2d4b371dae (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri Apr 26 10:59:38 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=3&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=3&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=4&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=4&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=5&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10970/files?page=5&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [Makefile, cdc/api/v2/api_helpers.go, cdc/api/v2/api_helpers_mock.go, cdc/api/v2/changefeed.go, cdc/api/v2/changefeed_test.go, cdc/api/v2/model.go, cdc/api/v2/model_test.go, cdc/api/v2/unsafe.go, cdc/entry/codec.go, cdc/entry/codec_test.go, cdc/entry/mounter.go, cdc/entry/mounter_test.go, cdc/entry/schema/snapshot.go, cdc/entry/schema/snapshot_test.go, cdc/entry/schema_storage.go, cdc/entry/schema_storage_test.go, cdc/entry/schema_test_helper.go, cdc/entry/validator.go, cdc/kv/client_bench_test.go, cdc/kv/client_test.go, cdc/kv/shared_client_test.go, cdc/kv/store_op.go, cdc/model/changefeed_test.go, cdc/model/codec/codec.go, cdc/model/codec/codec_test.go, cdc/model/codec/v1/codec.go, cdc/model/codec/v1/convert.go, cdc/model/codec/v1/convert_test.go, cdc/model/mounter.go, cdc/model/owner.go, cdc/model/schema_storage.go, cdc/model/schema_storage_test.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/owner/changefeed.go, cdc/owner/changefeed_test.go, cdc/owner/ddl_manager.go, cdc/owner/ddl_manager_test.go, cdc/owner/ddl_sink.go, cdc/owner/schema.go, cdc/owner/schema_test.go, cdc/processor/sourcemanager/engine/factory/factory.go, cdc/processor/sourcemanager/engine/factory/pebble.go, cdc/processor/sourcemanager/engine/pebble/db.go, cdc/processor/sourcemanager/engine/pebble/db_test.go, cdc/processor/sourcemanager/engine/pebble/event_sorter.go, cdc/processor/sourcemanager/engine/pebble/event_sorter_test.go, cdc/puller/ddl_puller.go, cdc/puller/ddl_puller_test.go, cdc/puller/puller.go, cdc/puller/puller_test.go, cdc/server/server.go, cdc/sink/ddlsink/cloudstorage/cloud_storage_ddl_sink.go, cdc/sink/ddlsink/cloudstorage/cloud_storage_ddl_sink_test.go, cdc/sink/ddlsink/mq/mq_ddl_sink_test.go, cdc/sink/ddlsink/mq/pulsar_ddl_sink_test.go, cdc/sink/ddlsink/mysql/mysql_ddl_sink.go, cdc/sink/ddlsink/mysql/mysql_ddl_sink_test.go, cdc/sink/dmlsink/cloudstorage/cloud_storage_dml_sink_test.go, cdc/sink/dmlsink/cloudstorage/defragmenter_test.go, cdc/sink/dmlsink/cloudstorage/dml_worker_test.go, cdc/sink/dmlsink/cloudstorage/encoding_worker_test.go, cdc/sink/dmlsink/mq/dispatcher/event_router.go, cdc/sink/dmlsink/mq/dispatcher/event_router_test.go, cdc/sink/dmlsink/mq/dispatcher/partition/columns_test.go, cdc/sink/dmlsink/mq/dispatcher/partition/index_value_test.go, cdc/sink/dmlsink/mq/transformer/columnselector/column_selector.go, cdc/sink/dmlsink/txn/event.go, cdc/sink/dmlsink/txn/event_test.go, cdc/sink/dmlsink/txn/mysql/dml.go, cdc/sink/dmlsink/txn/mysql/dml_test.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/cdc/main.go, cmd/dm-syncer/config.go, cmd/dm-worker/main.go, cmd/filter-helper/main.go, cmd/kafka-consumer/main.go, dm/chaos/cases/db.go, dm/chaos/cases/diff.go, dm/chaos/cases/schema.go, dm/chaos/cases/stmt.go, dm/chaos/cases/task.go, dm/checker/check_test.go, dm/checker/checker.go, dm/config/subtask.go, dm/config/subtask_test.go, dm/config/task.go, dm/config/task_converters.go, dm/config/task_converters_test.go, dm/config/task_test.go, dm/ctl/common/util.go, dm/dumpling/dumpling.go, dm/dumpling/dumpling_test.go, dm/loader/checkpoint.go, dm/loader/lightning.go, dm/loader/lightning_test.go, dm/master/config.go, dm/master/config_test.go, dm/master/openapi_view.go, dm/master/server.go, dm/master/server_test.go, dm/master/shardddl/optimist.go, dm/master/shardddl/optimist_test.go, dm/pkg/binlog/event/util.go, dm/pkg/checker/binlog.go, dm/pkg/checker/binlog_test.go, dm/pkg/checker/conn_checker.go, dm/pkg/checker/lightning.go, dm/pkg/checker/lightning_test.go, dm/pkg/checker/mysql_server.go, dm/pkg/checker/mysql_server_test.go, dm/pkg/checker/onlineddl.go, dm/pkg/checker/privilege.go, dm/pkg/checker/privilege_test.go, dm/pkg/checker/table_structure.go, dm/pkg/checker/table_structure_test.go, dm/pkg/checker/utils.go, dm/pkg/checker/utils_test.go, dm/pkg/conn/baseconn.go, dm/pkg/conn/baseconn_test.go, dm/pkg/conn/basedb.go, dm/pkg/conn/db.go, dm/pkg/conn/db_test.go, dm/pkg/conn/mockdb.go, dm/pkg/conn/utils.go, dm/pkg/election/election_test.go, dm/pkg/etcdutil/etcdutil_test.go, dm/pkg/log/log.go, dm/pkg/log/log_test.go, dm/pkg/parser/common.go, dm/pkg/parser/common_test.go, dm/pkg/retry/errors.go, dm/pkg/retry/strategy_test.go, dm/pkg/schema/tracker.go, dm/pkg/schema/tracker_test.go, dm/pkg/schema/visitor.go, dm/pkg/shardddl/optimism/info.go, dm/pkg/shardddl/optimism/info_test.go, dm/pkg/shardddl/optimism/keeper.go, dm/pkg/shardddl/optimism/keeper_test.go, dm/pkg/shardddl/optimism/lock.go, dm/pkg/shardddl/optimism/lock_test.go, dm/pkg/shardddl/pessimism/info_test.go, dm/pkg/upgrade/upgrade.go, dm/pkg/utils/common.go, dm/pkg/utils/common_test.go, dm/pkg/utils/time.go, dm/pkg/utils/util.go, dm/pkg/utils/util_test.go, dm/pkg/v1dbschema/schema.go, dm/relay/file_util.go, dm/relay/file_util_test.go, dm/relay/relay.go, dm/relay/relay_test.go, dm/simulator/config/config.go, dm/simulator/sqlgen/impl.go, dm/simulator/sqlgen/impl_test.go, dm/syncer/causality.go, dm/syncer/checkpoint.go, dm/syncer/checkpoint_flush_worker.go, dm/syncer/checkpoint_test.go, dm/syncer/compactor_test.go, dm/syncer/data_validator.go, dm/syncer/data_validator_test.go, dm/syncer/dbconn/db.go, dm/syncer/dbconn/upstream_db.go, dm/syncer/dbconn/utils.go, dm/syncer/ddl.go, dm/syncer/ddl_test.go, dm/syncer/dml.go, dm/syncer/dml_test.go, dm/syncer/dml_worker_test.go, dm/syncer/error.go, dm/syncer/error_test.go, dm/syncer/expr_filter_group.go, dm/syncer/expr_filter_group_test.go, dm/syncer/filter.go, dm/syncer/filter_test.go, dm/syncer/handle_error.go, dm/syncer/job.go, dm/syncer/job_test.go, dm/syncer/online-ddl-tools/online_ddl.go, dm/syncer/opt_sharding_group.go, dm/syncer/optimist.go, dm/syncer/safe-mode/mode.go, dm/syncer/safe-mode/mode_test.go, dm/syncer/schema.go, dm/syncer/shardddl/optimist.go, dm/syncer/shardddl/optimist_test.go, dm/syncer/sharding-meta/shardmeta.go, dm/syncer/sharding_group.go, dm/syncer/sharding_group_test.go, dm/syncer/status_test.go, dm/syncer/syncer.go, dm/syncer/syncer_test.go, dm/syncer/util.go, dm/syncer/util_test.go, dm/syncer/validate_worker.go, dm/syncer/validate_worker_test.go, dm/syncer/validator_checkpoint.go, dm/syncer/validator_checkpoint_test.go, dm/syncer/validator_cond.go, dm/syncer/validator_cond_test.go, dm/tests/incompatible_ddl_changes/conf/dm-task1.yaml, dm/tests/incompatible_ddl_changes/run.sh, dm/tests/lightning_mode/run.sh, dm/tests/tls/run.sh, dm/unit/unit_test.go, dm/worker/server_test.go, dm/worker/task_checker_test.go, engine/chaos/cases/dm/case.go, engine/chaos/cases/dm/db.go, engine/executor/server.go, engine/jobmaster/dm/checkpoint/agent.go, engine/jobmaster/dm/checkpoint/agent_test.go, engine/jobmaster/dm/config/config.go, engine/jobmaster/dm/ddl_coordinator.go, engine/pkg/meta/internal/etcdkv/mock.go, engine/pkg/meta/model/config.go, engine/pkg/orm/util_test.go, examples/golang/avro-checksum-verification/go.mod, examples/golang/avro-checksum-verification/go.sum, examples/golang/avro-checksum-verification/main.go, go.mod, go.sum, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/cmd/server/server.go, pkg/cmd/server/server_test.go, pkg/config/filter.go, pkg/config/outdated/v1.go, pkg/config/replica_config.go, pkg/errorutil/util.go, pkg/errorutil/util_test.go, pkg/etcd/etcd.go, pkg/filter/expr_filter.go, pkg/filter/expr_filter_test.go, pkg/filter/filter.go, pkg/filter/filter_test.go, pkg/filter/filter_test_helper.go, pkg/filter/sql_event_filter.go, pkg/filter/utils.go, pkg/filter/utils_test.go, pkg/httputil/httputil_test.go, pkg/migrate/migrate_test.go, pkg/pdutil/api_client.go, pkg/pdutil/api_client_test.go, pkg/pdutil/utils_test.go, pkg/sink/cloudstorage/path_test.go, pkg/sink/cloudstorage/table_definition.go, pkg/sink/cloudstorage/table_definition_test.go, pkg/sink/codec/avro/avro.go, pkg/sink/codec/avro/avro_test.go, pkg/sink/codec/avro/decoder.go, pkg/sink/codec/avro/decoder_test.go, pkg/sink/codec/canal/canal_encoder_test.go, pkg/sink/codec/canal/canal_entry.go, pkg/sink/codec/canal/canal_entry_test.go, pkg/sink/codec/canal/canal_json_message.go, pkg/sink/codec/canal/canal_json_row_event_encoder_test.go, pkg/sink/codec/canal/canal_json_txn_event_encoder_test.go, pkg/sink/codec/canal/canal_test_util.go, pkg/sink/codec/common/message_test.go, pkg/sink/codec/common/verify_checksum.go, pkg/sink/codec/craft/craft_encoder_test.go, pkg/sink/codec/craft/message_decoder.go, pkg/sink/codec/craft/message_encoder.go, pkg/sink/codec/csv/csv_decoder.go, pkg/sink/codec/csv/csv_decoder_test.go, pkg/sink/codec/csv/csv_encoder_test.go, pkg/sink/codec/csv/csv_message.go, pkg/sink/codec/csv/csv_message_test.go, pkg/sink/codec/internal/batch_tester.go, pkg/sink/codec/internal/column.go, pkg/sink/codec/internal/java.go, pkg/sink/codec/maxwell/maxwell_encoder_test.go, pkg/sink/codec/maxwell/maxwell_message.go, pkg/sink/codec/maxwell/maxwell_message_test.go, pkg/sink/codec/open/open_protocol_decoder.go, pkg/sink/codec/open/open_protocol_encoder_test.go, pkg/sink/codec/open/open_protocol_message.go, pkg/sink/codec/open/open_protocol_message_test.go, pkg/sink/kafka/claimcheck/claim_check_test.go, pkg/sink/mysql/db_helper.go, pkg/spanz/span.go, pkg/spanz/span_test.go, pkg/sqlmodel/causality.go, pkg/sqlmodel/row_change.go, pkg/sqlmodel/row_change_test.go, pkg/sqlmodel/utils.go, pkg/sqlmodel/where_handle.go, pkg/sqlmodel/where_handle_test.go, pkg/txnutil/gc/gc_manager.go, pkg/txnutil/gc/gc_manager_test.go, pkg/upstream/upstream.go, pkg/util/memory.go, pkg/util/tz.go, pkg/version/check.go, pkg/version/check_test.go, tests/integration_tests/cdc/dailytest/db.go, tests/integration_tests/cdc/dailytest/job.go, tests/integration_tests/cdc/dailytest/parser.go, tests/integration_tests/resolve_lock/main.go, tests/integration_tests/util/db.go] [Pipeline] echo diff file matched: dm/chaos/cases/db.go [Pipeline] echo matched, some diff files full path start with dm/ or pkg/ or go.mod, run the dm integration test [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-823a389-aa323aa) 199979520 bytes in 1.76 secs (113583556 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/.git/ .git HEAD is now at 8b79396bb Merge commit 'aa323aa99660421afc797ea115dbb30242e573cf' into HEAD POST git-upload-pack (686 bytes) From https://github.com/pingcap/tiflow = [up to date] release-7.5 -> origin/release-7.5 = [up to date] refs/pull/10970/head -> origin/pr/10970/head Warning: you are leaving 1 commit behind, not connected to any of your branches: 8b79396bb Merge commit 'aa323aa99660421afc797ea115dbb30242e573cf' into HEAD If you want to keep it by creating a new branch, this may be a good time to do so with: git branch 8b79396bb HEAD is now at 823a3899c sink(ticdc): Revert changes related to the conflict detector (#10924) ๐Ÿšง Checkouting to base SHA:823a3899cc17bf14ad7875ab0cc69092186abc4a... HEAD is now at 823a3899c sink(ticdc): Revert changes related to the conflict detector (#10924) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 823a3899cc17bf14ad7875ab0cc69092186abc4a 823a3899c sink(ticdc): Revert changes related to the conflict detector (#10924) 3c4a12d87 mysql(ticdc): fix the data race in the conflict detector (#10630) (#10642) 0d6a6613d kvclient(ticdc): close the grpc client after all goroutine exited to prevent data race and panic (#10865) (#10906) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 823a3899cc17bf14ad7875ab0cc69092186abc4a ... Merge made by the 'ort' strategy. Makefile | 2 +- cdc/api/v2/api_helpers.go | 2 +- cdc/api/v2/api_helpers_mock.go | 2 +- cdc/api/v2/changefeed.go | 2 +- cdc/api/v2/changefeed_test.go | 2 +- cdc/api/v2/model.go | 2 +- cdc/api/v2/model_test.go | 2 +- cdc/api/v2/unsafe.go | 2 +- cdc/entry/codec.go | 12 +- cdc/entry/codec_test.go | 6 +- cdc/entry/mounter.go | 38 +- cdc/entry/mounter_test.go | 60 +- cdc/entry/schema/snapshot.go | 4 +- cdc/entry/schema/snapshot_test.go | 2 +- cdc/entry/schema_storage.go | 4 +- cdc/entry/schema_storage_test.go | 24 +- cdc/entry/schema_test_helper.go | 18 +- cdc/entry/validator.go | 2 +- cdc/kv/client_bench_test.go | 2 +- cdc/kv/client_test.go | 4 +- cdc/kv/shared_client_test.go | 2 +- cdc/kv/store_op.go | 6 +- cdc/model/changefeed_test.go | 2 +- cdc/model/codec/codec.go | 2 +- cdc/model/codec/codec_test.go | 6 +- cdc/model/codec/v1/codec.go | 4 +- cdc/model/codec/v1/convert.go | 2 +- cdc/model/codec/v1/convert_test.go | 6 +- cdc/model/mounter.go | 2 +- cdc/model/owner.go | 2 +- cdc/model/schema_storage.go | 10 +- cdc/model/schema_storage_test.go | 6 +- cdc/model/sink.go | 6 +- cdc/model/sink_test.go | 6 +- cdc/owner/changefeed.go | 2 +- cdc/owner/changefeed_test.go | 2 +- cdc/owner/ddl_manager.go | 2 +- cdc/owner/ddl_manager_test.go | 2 +- cdc/owner/ddl_sink.go | 6 +- cdc/owner/schema.go | 6 +- cdc/owner/schema_test.go | 21 +- .../sourcemanager/engine/factory/factory.go | 6 +- .../sourcemanager/engine/factory/pebble.go | 39 +- cdc/processor/sourcemanager/engine/pebble/db.go | 13 +- .../sourcemanager/engine/pebble/db_test.go | 1 + .../sourcemanager/engine/pebble/event_sorter.go | 6 +- .../engine/pebble/event_sorter_test.go | 8 +- cdc/puller/ddl_puller.go | 6 +- cdc/puller/ddl_puller_test.go | 6 +- cdc/puller/puller.go | 2 +- cdc/puller/puller_test.go | 4 +- cdc/server/server.go | 2 +- .../ddlsink/cloudstorage/cloud_storage_ddl_sink.go | 2 +- .../cloudstorage/cloud_storage_ddl_sink_test.go | 6 +- cdc/sink/ddlsink/mq/mq_ddl_sink_test.go | 2 +- cdc/sink/ddlsink/mq/pulsar_ddl_sink_test.go | 2 +- cdc/sink/ddlsink/mysql/mysql_ddl_sink.go | 2 +- cdc/sink/ddlsink/mysql/mysql_ddl_sink_test.go | 4 +- .../cloudstorage/cloud_storage_dml_sink_test.go | 8 +- cdc/sink/dmlsink/cloudstorage/defragmenter_test.go | 6 +- cdc/sink/dmlsink/cloudstorage/dml_worker_test.go | 6 +- .../dmlsink/cloudstorage/encoding_worker_test.go | 6 +- cdc/sink/dmlsink/mq/dispatcher/event_router.go | 2 +- .../dmlsink/mq/dispatcher/event_router_test.go | 2 +- .../mq/dispatcher/partition/columns_test.go | 2 +- .../mq/dispatcher/partition/index_value_test.go | 2 +- .../transformer/columnselector/column_selector.go | 2 +- cdc/sink/dmlsink/txn/event.go | 2 +- cdc/sink/dmlsink/txn/event_test.go | 2 +- cdc/sink/dmlsink/txn/mysql/dml.go | 2 +- cdc/sink/dmlsink/txn/mysql/dml_test.go | 4 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 12 +- cmd/cdc/main.go | 2 +- cmd/dm-syncer/config.go | 4 +- cmd/dm-worker/main.go | 3 + cmd/filter-helper/main.go | 2 +- cmd/kafka-consumer/main.go | 2 +- dm/chaos/cases/db.go | 4 +- dm/chaos/cases/diff.go | 2 +- dm/chaos/cases/schema.go | 4 +- dm/chaos/cases/stmt.go | 4 +- dm/chaos/cases/task.go | 4 +- dm/checker/check_test.go | 4 +- dm/checker/checker.go | 50 +- dm/config/subtask.go | 8 +- dm/config/subtask_test.go | 2 +- dm/config/task.go | 8 +- dm/config/task_converters.go | 4 +- dm/config/task_converters_test.go | 2 +- dm/config/task_test.go | 4 +- dm/ctl/common/util.go | 2 +- dm/dumpling/dumpling.go | 6 +- dm/dumpling/dumpling_test.go | 6 +- dm/loader/checkpoint.go | 2 +- dm/loader/lightning.go | 46 +- dm/loader/lightning_test.go | 4 +- dm/master/config.go | 10 +- dm/master/config_test.go | 16 +- dm/master/openapi_view.go | 2 +- dm/master/server.go | 2 +- dm/master/server_test.go | 14 +- dm/master/shardddl/optimist.go | 2 +- dm/master/shardddl/optimist_test.go | 14 +- dm/pkg/binlog/event/util.go | 6 +- dm/pkg/checker/binlog.go | 5 +- dm/pkg/checker/binlog_test.go | 2 +- dm/pkg/checker/conn_checker.go | 4 +- dm/pkg/checker/lightning.go | 20 +- dm/pkg/checker/lightning_test.go | 2 +- dm/pkg/checker/mysql_server.go | 2 +- dm/pkg/checker/mysql_server_test.go | 2 +- dm/pkg/checker/onlineddl.go | 4 +- dm/pkg/checker/privilege.go | 14 +- dm/pkg/checker/privilege_test.go | 4 +- dm/pkg/checker/table_structure.go | 15 +- dm/pkg/checker/table_structure_test.go | 2 +- dm/pkg/checker/utils.go | 6 +- dm/pkg/checker/utils_test.go | 4 +- dm/pkg/conn/baseconn.go | 2 +- dm/pkg/conn/baseconn_test.go | 2 +- dm/pkg/conn/basedb.go | 4 +- dm/pkg/conn/db.go | 10 +- dm/pkg/conn/db_test.go | 8 +- dm/pkg/conn/mockdb.go | 16 +- dm/pkg/conn/utils.go | 2 +- dm/pkg/election/election_test.go | 8 +- dm/pkg/etcdutil/etcdutil_test.go | 20 +- dm/pkg/log/log.go | 4 +- dm/pkg/log/log_test.go | 4 +- dm/pkg/parser/common.go | 12 +- dm/pkg/parser/common_test.go | 4 +- dm/pkg/retry/errors.go | 2 +- dm/pkg/retry/strategy_test.go | 2 +- dm/pkg/schema/tracker.go | 43 +- dm/pkg/schema/tracker_test.go | 14 +- dm/pkg/schema/visitor.go | 4 +- dm/pkg/shardddl/optimism/info.go | 4 +- dm/pkg/shardddl/optimism/info_test.go | 12 +- dm/pkg/shardddl/optimism/keeper.go | 2 +- dm/pkg/shardddl/optimism/keeper_test.go | 6 +- dm/pkg/shardddl/optimism/lock.go | 10 +- dm/pkg/shardddl/optimism/lock_test.go | 10 +- dm/pkg/shardddl/pessimism/info_test.go | 2 +- dm/pkg/upgrade/upgrade.go | 2 +- dm/pkg/utils/common.go | 41 +- dm/pkg/utils/common_test.go | 2 +- dm/pkg/utils/time.go | 6 +- dm/pkg/utils/util.go | 2 +- dm/pkg/utils/util_test.go | 2 +- dm/pkg/v1dbschema/schema.go | 4 +- dm/relay/file_util.go | 2 +- dm/relay/file_util_test.go | 2 +- dm/relay/relay.go | 5 +- dm/relay/relay_test.go | 2 +- dm/simulator/config/config.go | 2 +- dm/simulator/sqlgen/impl.go | 10 +- dm/simulator/sqlgen/impl_test.go | 4 +- dm/syncer/causality.go | 2 +- dm/syncer/checkpoint.go | 8 +- dm/syncer/checkpoint_flush_worker.go | 2 +- dm/syncer/checkpoint_test.go | 8 +- dm/syncer/compactor_test.go | 4 +- dm/syncer/data_validator.go | 4 +- dm/syncer/data_validator_test.go | 6 +- dm/syncer/dbconn/db.go | 2 +- dm/syncer/dbconn/upstream_db.go | 6 +- dm/syncer/dbconn/utils.go | 4 +- dm/syncer/ddl.go | 58 +- dm/syncer/ddl_test.go | 10 +- dm/syncer/dml.go | 12 +- dm/syncer/dml_test.go | 16 +- dm/syncer/dml_worker_test.go | 10 +- dm/syncer/error.go | 16 +- dm/syncer/error_test.go | 2 +- dm/syncer/expr_filter_group.go | 20 +- dm/syncer/expr_filter_group_test.go | 6 +- dm/syncer/filter.go | 2 +- dm/syncer/filter_test.go | 4 +- dm/syncer/handle_error.go | 4 +- dm/syncer/job.go | 2 +- dm/syncer/job_test.go | 2 +- dm/syncer/online-ddl-tools/online_ddl.go | 10 +- dm/syncer/opt_sharding_group.go | 2 +- dm/syncer/optimist.go | 2 +- dm/syncer/safe-mode/mode.go | 2 +- dm/syncer/safe-mode/mode_test.go | 2 +- dm/syncer/schema.go | 14 +- dm/syncer/shardddl/optimist.go | 4 +- dm/syncer/shardddl/optimist_test.go | 12 +- dm/syncer/sharding-meta/shardmeta.go | 4 +- dm/syncer/sharding_group.go | 4 +- dm/syncer/sharding_group_test.go | 4 +- dm/syncer/status_test.go | 4 +- dm/syncer/syncer.go | 18 +- dm/syncer/syncer_test.go | 14 +- dm/syncer/util.go | 7 +- dm/syncer/util_test.go | 10 +- dm/syncer/validate_worker.go | 8 +- dm/syncer/validate_worker_test.go | 10 +- dm/syncer/validator_checkpoint.go | 4 +- dm/syncer/validator_checkpoint_test.go | 6 +- dm/syncer/validator_cond.go | 2 +- dm/syncer/validator_cond_test.go | 10 +- .../incompatible_ddl_changes/conf/dm-task1.yaml | 28 +- dm/tests/incompatible_ddl_changes/run.sh | 96 +- dm/tests/lightning_mode/run.sh | 2 +- dm/tests/tls/run.sh | 9 +- dm/unit/unit_test.go | 4 +- dm/worker/server_test.go | 8 +- dm/worker/task_checker_test.go | 2 +- engine/chaos/cases/dm/case.go | 4 +- engine/chaos/cases/dm/db.go | 2 +- engine/executor/server.go | 4 +- engine/jobmaster/dm/checkpoint/agent.go | 14 +- engine/jobmaster/dm/checkpoint/agent_test.go | 7 +- engine/jobmaster/dm/config/config.go | 4 +- engine/jobmaster/dm/ddl_coordinator.go | 10 +- engine/pkg/meta/internal/etcdkv/mock.go | 4 +- engine/pkg/meta/model/config.go | 6 +- engine/pkg/orm/util_test.go | 2 +- examples/golang/avro-checksum-verification/go.mod | 79 +- examples/golang/avro-checksum-verification/go.sum | 311 +++-- examples/golang/avro-checksum-verification/main.go | 23 +- go.mod | 276 ++--- go.sum | 1227 ++++++-------------- pkg/applier/redo.go | 2 +- pkg/applier/redo_test.go | 2 +- pkg/cmd/server/server.go | 2 +- pkg/cmd/server/server_test.go | 2 +- pkg/config/filter.go | 2 +- pkg/config/outdated/v1.go | 2 +- pkg/config/replica_config.go | 2 +- pkg/errorutil/util.go | 8 +- pkg/errorutil/util_test.go | 4 +- pkg/etcd/etcd.go | 4 +- pkg/filter/expr_filter.go | 24 +- pkg/filter/expr_filter_test.go | 14 +- pkg/filter/filter.go | 4 +- pkg/filter/filter_test.go | 2 +- pkg/filter/filter_test_helper.go | 18 +- pkg/filter/sql_event_filter.go | 6 +- pkg/filter/utils.go | 8 +- pkg/filter/utils_test.go | 18 +- pkg/httputil/httputil_test.go | 2 +- pkg/migrate/migrate_test.go | 4 +- pkg/pdutil/api_client.go | 6 +- pkg/pdutil/api_client_test.go | 6 +- pkg/pdutil/utils_test.go | 6 +- pkg/sink/cloudstorage/path_test.go | 6 +- pkg/sink/cloudstorage/table_definition.go | 8 +- pkg/sink/cloudstorage/table_definition_test.go | 8 +- pkg/sink/codec/avro/avro.go | 8 +- pkg/sink/codec/avro/avro_test.go | 6 +- pkg/sink/codec/avro/decoder.go | 32 +- pkg/sink/codec/avro/decoder_test.go | 15 +- pkg/sink/codec/canal/canal_encoder_test.go | 2 +- pkg/sink/codec/canal/canal_entry.go | 6 +- pkg/sink/codec/canal/canal_entry_test.go | 4 +- pkg/sink/codec/canal/canal_json_message.go | 4 +- .../canal/canal_json_row_event_encoder_test.go | 2 +- .../canal/canal_json_txn_event_encoder_test.go | 2 +- pkg/sink/codec/canal/canal_test_util.go | 4 +- pkg/sink/codec/common/message_test.go | 6 +- pkg/sink/codec/common/verify_checksum.go | 190 +++ pkg/sink/codec/craft/craft_encoder_test.go | 2 +- pkg/sink/codec/craft/message_decoder.go | 4 +- pkg/sink/codec/craft/message_encoder.go | 2 +- pkg/sink/codec/csv/csv_decoder.go | 6 +- pkg/sink/codec/csv/csv_decoder_test.go | 6 +- pkg/sink/codec/csv/csv_encoder_test.go | 6 +- pkg/sink/codec/csv/csv_message.go | 10 +- pkg/sink/codec/csv/csv_message_test.go | 10 +- pkg/sink/codec/internal/batch_tester.go | 2 +- pkg/sink/codec/internal/column.go | 2 +- pkg/sink/codec/internal/java.go | 2 +- pkg/sink/codec/maxwell/maxwell_encoder_test.go | 4 +- pkg/sink/codec/maxwell/maxwell_message.go | 4 +- pkg/sink/codec/maxwell/maxwell_message_test.go | 2 +- pkg/sink/codec/open/open_protocol_decoder.go | 4 +- pkg/sink/codec/open/open_protocol_encoder_test.go | 4 +- pkg/sink/codec/open/open_protocol_message.go | 2 +- pkg/sink/codec/open/open_protocol_message_test.go | 2 +- pkg/sink/kafka/claimcheck/claim_check_test.go | 2 +- pkg/sink/mysql/db_helper.go | 4 +- pkg/spanz/span.go | 8 +- pkg/spanz/span_test.go | 2 +- pkg/sqlmodel/causality.go | 8 +- pkg/sqlmodel/row_change.go | 4 +- pkg/sqlmodel/row_change_test.go | 13 +- pkg/sqlmodel/utils.go | 2 +- pkg/sqlmodel/where_handle.go | 6 +- pkg/sqlmodel/where_handle_test.go | 6 +- pkg/txnutil/gc/gc_manager.go | 8 +- pkg/txnutil/gc/gc_manager_test.go | 2 +- pkg/upstream/upstream.go | 2 +- pkg/util/memory.go | 2 +- pkg/util/tz.go | 31 +- pkg/version/check.go | 2 +- pkg/version/check_test.go | 2 +- tests/integration_tests/cdc/dailytest/db.go | 2 +- tests/integration_tests/cdc/dailytest/job.go | 4 +- tests/integration_tests/cdc/dailytest/parser.go | 8 +- tests/integration_tests/resolve_lock/main.go | 8 +- tests/integration_tests/util/db.go | 2 +- 305 files changed, 2043 insertions(+), 2191 deletions(-) create mode 100644 pkg/sink/codec/common/verify_checksum.go ๐Ÿงพ Pre-merged result: 3ca0c6ff17a09c4824dcb351353e4007cb572d5c 3ca0c6ff1 Merge commit 'aa323aa99660421afc797ea115dbb30242e573cf' into HEAD aa323aa99 fix potential typos. 13c5e9410 update โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-823a389-aa323aa already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./dm/tests/download-integration-test-binaries.sh release-7.5 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 147 0 --:--:-- --:--:-- --:--:-- 147 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 139 0 --:--:-- --:--:-- --:--:-- 139 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 649 0 --:--:-- --:--:-- --:--:-- 650 100 41 100 41 0 0 649 0 --:--:-- --:--:-- --:--:-- 640 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1084 0 --:--:-- --:--:-- --:--:-- 1108 Download binaries... >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/e406d5780b18a1f2aaf6230cde4b6403991e228d/centos7/tidb-server.tar.gz 2024-04-26 19:00:04 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/e406d5780b18a1f2aaf6230cde4b6403991e228d/centos7/tidb-server.tar.gz [511580034/511580034] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/58453365285465cd90bc4472cff2bad7ce4d764b/centos7/pd-server.tar.gz 2024-04-26 19:00:17 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/58453365285465cd90bc4472cff2bad7ce4d764b/centos7/pd-server.tar.gz [173219013/173219013] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990/centos7/tikv-server.tar.gz 2024-04-26 19:00:41 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990/centos7/tikv-server.tar.gz [810252984/810252984] -> "tmp/tikv-server.tar.gz" [1] >>> download tidb-tools.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/d28ee6cd546e41862cf361e7a35552fb517b883e/centos7/tidb-tools.tar.gz 2024-04-26 19:00:57 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/d28ee6cd546e41862cf361e7a35552fb517b883e/centos7/tidb-tools.tar.gz [185439936/185439936] -> "tmp/tidb-tools.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-04-26 19:01:00 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download gh-ost-binary-linux-20200828140552.tar.gz from https://github.com/github/gh-ost/releases/download/v1.1.0/gh-ost-binary-linux-20200828140552.tar.gz 2024-04-26 19:01:05 URL:https://objects.githubusercontent.com/github-production-release-asset-2e65be/54378638/ac8d1400-e94a-11ea-9664-8b516d434420?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240426%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240426T110101Z&X-Amz-Expires=300&X-Amz-Signature=6fbc79942a4056e7fa930514467a62fc3347eaab639fa5a34cb8d7ea7bc8cf1c&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=54378638&response-content-disposition=attachment%3B%20filename%3Dgh-ost-binary-linux-20200828140552.tar.gz&response-content-type=application%2Foctet-stream [5141587/5141587] -> "tmp/gh-ost-binary-linux-20200828140552.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.1G drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:01 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:01 .. drwxr-sr-x 2 jenkins jenkins 4.0K Apr 26 19:01 bin -rwxr-xr-x 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 33M Apr 3 18:05 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M Apr 3 18:05 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M Apr 3 18:05 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 3 18:05 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M Apr 3 18:04 pd-server -rwxr-xr-x 1 jenkins jenkins 25M Apr 3 18:05 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 3 18:05 regions-dump -rwxr-xr-x 1 jenkins jenkins 29M Apr 3 18:05 stores-dump -rwxr-xr-x 1 jenkins jenkins 127M Apr 8 13:50 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M Apr 22 17:56 tidb-server -rwxr-xr-x 1 jenkins jenkins 353M Apr 25 18:18 tikv-server + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/gh-ost ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.1G drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:01 . drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:01 .. drwxr-sr-x 2 jenkins jenkins 4.0K Apr 26 19:01 bin -rwxr-xr-x 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 33M Apr 3 18:05 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M Apr 3 18:05 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M Apr 3 18:05 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 3 18:05 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M Apr 3 18:04 pd-server -rwxr-xr-x 1 jenkins jenkins 25M Apr 3 18:05 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 3 18:05 regions-dump -rwxr-xr-x 1 jenkins jenkins 29M Apr 3 18:05 stores-dump -rwxr-xr-x 1 jenkins jenkins 127M Apr 8 13:50 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M Apr 22 17:56 tidb-server -rwxr-xr-x 1 jenkins jenkins 353M Apr 25 18:18 tikv-server + ./bin/tidb-server -V Release Version: v7.5.1-43-ge406d5780b Edition: Community Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d Git Branch: release-7.5 UTC Build Time: 2024-04-22 09:56:10 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 + ./bin/tikv-server -V TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990 Git Commit Branch: release-7.5 UTC Build Time: 2024-04-25 09:30:46 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + [[ ! -f bin/dm-master.test ]] + echo 'Building binaries...' Building binaries... + make dm_integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240425142112-54ba0ed68407 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/pd v1.1.0-beta.0.20230203015356-248b3f0be132 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gin-gonic/gin v1.8.1 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240425142112-54ba0ed68407 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/failpoint v0.0.0-20240412033321-fd0796e60f86 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240424052342-0229f4077f0c go: downloading golang.org/x/time v0.5.0 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240408054520-d28ee6cd546e go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading blainsmith.com/go/seahash v1.2.1 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading cloud.google.com/go/storage v1.36.0 go: downloading github.com/swaggo/swag v1.8.3 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading cloud.google.com/go v0.112.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240304212257-790db918fca8 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/jsonreference v0.20.2 go: downloading github.com/go-openapi/spec v0.20.6 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/klauspost/compress v1.17.7 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20230619063055-fd67d94318fd go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.8 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading golang.org/x/crypto v0.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.24.0 go: downloading cloud.google.com/go/iam v1.1.6 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.162.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/robfig/cron v1.2.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240308144416-29370a3891b7 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/spf13/cobra v1.8.0 go: downloading gorm.io/gorm v1.23.8 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading go.uber.org/dig v1.13.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/glebarez/go-sqlite v1.17.3 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/glebarez/sqlite v1.4.6 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/fatih/color v1.16.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/go-playground/validator/v10 v10.11.1 go: downloading github.com/pelletier/go-toml/v2 v2.0.5 go: downloading github.com/ugorji/go/codec v1.2.7 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/go-openapi/jsonpointer v0.19.6 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/go-openapi/swag v0.22.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.22.0 go: downloading go.opentelemetry.io/otel/trace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.16.8 go: downloading modernc.org/sqlite v1.17.3 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-playground/universal-translator v0.18.0 go: downloading github.com/leodido/go-urn v1.2.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/danjacques/gofslock v0.0.0-20220131014315-6e321f4509c8 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/go-playground/locales v0.14.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/rivo/uniseg v0.4.6 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading modernc.org/mathutil v1.6.0 go: downloading modernc.org/memory v1.1.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 go: downloading github.com/felixge/httpsnoop v1.0.4 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g3ca0c6ff1" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:01:06" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3ca0c6ff17a09c4824dcb351353e4007cb572d5c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g3ca0c6ff1"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dm-worker.test github.com/pingcap/tiflow/cmd/dm-worker \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g3ca0c6ff1" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:01:06" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3ca0c6ff17a09c4824dcb351353e4007cb572d5c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g3ca0c6ff1"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dm-master.test github.com/pingcap/tiflow/cmd/dm-master \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=1 GO111MODULE=on go test -p 3 -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g3ca0c6ff1" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:01:06" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3ca0c6ff17a09c4824dcb351353e4007cb572d5c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g3ca0c6ff1"' -c -cover -covermode=count \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dmctl.test github.com/pingcap/tiflow/cmd/dm-ctl \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v7.5.1-57-g3ca0c6ff1" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-04-26 11:01:06" -X "github.com/pingcap/tiflow/pkg/version.GitHash=3ca0c6ff17a09c4824dcb351353e4007cb572d5c" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v7.5.1-57-g3ca0c6ff1"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dm-syncer.test github.com/pingcap/tiflow/cmd/dm-syncer \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) ./dm/tests/prepare_tools.sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + mkdir -p bin/dm-test-tools + cp -r ./dm/tests/bin/check_exit_safe_binlog ./dm/tests/bin/check_master_http_apis ./dm/tests/bin/check_master_online ./dm/tests/bin/check_master_online_http ./dm/tests/bin/check_worker_online ./bin/dm-test-tools + ls -alh ./bin total 1.1G drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:09 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:01 .. -rwxr-xr-x 1 jenkins jenkins 295M Apr 26 19:05 dm-master.test -rwxr-xr-x 1 jenkins jenkins 282M Apr 26 19:08 dm-syncer.test drwxr-sr-x 2 jenkins jenkins 4.0K Apr 26 19:09 dm-test-tools -rwxr-xr-x 1 jenkins jenkins 292M Apr 26 19:05 dm-worker.test -rwxr-xr-x 1 jenkins jenkins 213M Apr 26 19:08 dmctl.test + ls -alh ./bin/dm-test-tools total 338M drwxr-sr-x 2 jenkins jenkins 4.0K Apr 26 19:09 . drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:09 .. -rwxr-xr-x 1 jenkins jenkins 202M Apr 26 19:09 check_exit_safe_binlog -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_master_http_apis -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_master_online -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_master_online_http -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_worker_online + which ./bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test + which ./bin/dm-syncer.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test + which ./bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test + which ./bin/dmctl.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dmctl.test + which ./bin/dm-test-tools/check_master_online /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_master_online + which ./bin/dm-test-tools/check_worker_online /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_worker_online [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/dm-integration-test/rev-823a389-aa323aa) 1487542272 bytes in 41.58 secs (35777138 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/gh-ost ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.1G drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:09 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:01 .. drwxr-sr-x 2 jenkins jenkins 4.0K Apr 26 19:09 bin -rwxr-xr-x 1 jenkins jenkins 295M Apr 26 19:05 dm-master.test -rwxr-xr-x 1 jenkins jenkins 282M Apr 26 19:08 dm-syncer.test drwxr-sr-x 2 jenkins jenkins 4.0K Apr 26 19:09 dm-test-tools -rwxr-xr-x 1 jenkins jenkins 292M Apr 26 19:05 dm-worker.test -rwxr-xr-x 1 jenkins jenkins 213M Apr 26 19:08 dmctl.test -rwxr-xr-x 1 jenkins jenkins 9.8M Apr 26 19:09 gh-ost -rwxr-xr-x 1 jenkins jenkins 50M Apr 26 19:09 minio -rwxr-xr-x 1 jenkins jenkins 33M Apr 26 19:09 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 42M Apr 26 19:09 pd-ctl -rwxr-xr-x 1 jenkins jenkins 30M Apr 26 19:09 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 26 19:09 pd-recover -rwxr-xr-x 1 jenkins jenkins 102M Apr 26 19:09 pd-server -rwxr-xr-x 1 jenkins jenkins 25M Apr 26 19:09 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 29M Apr 26 19:09 regions-dump -rwxr-xr-x 1 jenkins jenkins 29M Apr 26 19:09 stores-dump -rwxr-xr-x 1 jenkins jenkins 127M Apr 26 19:09 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 199M Apr 26 19:09 tidb-server -rwxr-xr-x 1 jenkins jenkins 353M Apr 26 19:09 tikv-server + ls -alh ./bin/dm-test-tools total 338M drwxr-sr-x 2 jenkins jenkins 4.0K Apr 26 19:09 . drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:09 .. -rwxr-xr-x 1 jenkins jenkins 202M Apr 26 19:09 check_exit_safe_binlog -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_master_http_apis -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_master_online -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_master_online_http -rwxr-xr-x 1 jenkins jenkins 34M Apr 26 19:09 check_worker_online [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 95.25 secs (33066082 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd-tzt6q Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b-mc39j [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8v Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. Created Pod: kubernetes jenkins-tiflow/dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0-r94xj [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44-chjzs-d48dd Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv-2jkqf [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4-hvg39 Still waiting to schedule task โ€˜dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd-tzt6qโ€™ is offline Created Pod: kubernetes jenkins-tiflow/dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns-jfrv4 Still waiting to schedule task โ€˜dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b-mc39jโ€™ is offline Still waiting to schedule task โ€˜dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhplโ€™ is offline Agent dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44-chjzs-d48dd is provisioned from template dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44-chjzs --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8a992abc87e466f620af8983517275f25734f7f5" jenkins/label: "dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44" name: "dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44-chjzs-d48dd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44-chjzs-d48dd" - name: "JENKINS_NAME" value: "dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44-chjzs-d48dd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-2d1f28c9-18b6-4daa-b96b-3ee287f8ec44-chjzs-d48dd in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Still waiting to schedule task โ€˜dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8vโ€™ is offline No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@25e07db9; decorates RemoteLauncher[hudson.remoting.Channel@55e121ba:JNLP4-connect connection from 10.233.107.163/10.233.107.163:54700] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Still waiting to schedule task โ€˜dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4dโ€™ is offline Still waiting to schedule task โ€˜dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0-r94xjโ€™ is offline > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" [Pipeline] withEnv [Pipeline] { [Pipeline] container > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh Still waiting to schedule task โ€˜dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv-2jkqfโ€™ is offline + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K Apr 26 11:11 . drwxr-sr-x 17 1000 1000 4.0K Apr 26 11:11 .. -rw------- 1 root 1000 1.7K Apr 26 11:11 ca-key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:11 ca.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:11 client-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:11 client-key.pem -rw------- 1 root 1000 1.7K Apr 26 11:11 private_key.pem -rw-r--r-- 1 root 1000 451 Apr 26 11:11 public_key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:11 server-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:11 server-key.pem [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Still waiting to schedule task โ€˜dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns-jfrv4โ€™ is offline Still waiting to schedule task โ€˜dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9hโ€™ is offline Created Pod: kubernetes jenkins-tiflow/dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2 Still waiting to schedule task โ€˜dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglkโ€™ is offline Still waiting to schedule task โ€˜dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2โ€™ is offline Still waiting to schedule task โ€˜dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4-hvg39โ€™ is offline Agent dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk is provisioned from template dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8aa08094fda0abfbd35e0689cec2a41133617e39" jenkins/label: "dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2" name: "dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk" - name: "JENKINS_NAME" value: "dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Agent dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b-mc39j is provisioned from template dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b71f7c8adabce88bf91808204fad3b47b1a1123e" jenkins/label: "dm-it-d548b765-b287-446a-8015-5ac042f2d9ad" name: "dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b-mc39j" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b-mc39j" - name: "JENKINS_NAME" value: "dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b-mc39j" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-d548b765-b287-446a-8015-5ac042f2d9ad-9xv9b-mc39j in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 8.72 secs (361088807 bytes/sec) [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@53dd3108; decorates RemoteLauncher[hudson.remoting.Channel@1110dfb7:JNLP4-connect connection from 10.233.105.203/10.233.105.203:51308] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@69eeb61a; decorates RemoteLauncher[hudson.remoting.Channel@3a28e429:JNLP4-connect connection from 10.233.100.81/10.233.100.81:60080] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv-2jkqf is provisioned from template dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8529a0ba5749fcc146e4099b44310baeccc9930f" jenkins/label: "dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8" name: "dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv-2jkqf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv-2jkqf" - name: "JENKINS_NAME" value: "dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv-2jkqf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:12 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:12 .. -rw-r--r-- 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 Apr 26 19:12 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:12 .git drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 .github -rw-r--r-- 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r-- 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r-- 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:12 bin drwxr-sr-x 17 jenkins jenkins 4.0K Apr 26 19:12 cdc drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:12 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K Apr 26 19:12 cmd drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 deployments drwxr-sr-x 26 jenkins jenkins 4.0K Apr 26 19:12 dm drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:12 docs drwxr-sr-x 13 jenkins jenkins 4.0K Apr 26 19:12 engine -rwxr-xr-x 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 examples -rw-r--r-- 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r-- 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 metrics drwxr-sr-x 45 jenkins jenkins 4.0K Apr 26 19:12 pkg drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:12 proto drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:12 scripts drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:12 tests -rw-r--r-- 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break Running on dm-it-dbd1771d-3075-4ffb-9b03-554f039ad9b8-rcdrv-2jkqf in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] } [Pipeline] { [Pipeline] // timeout [Pipeline] sh [Pipeline] checkout The recommended git tool is: git + '[' TLS_GROUP == G06 ']' + echo 'run G06 test' run G06 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G06 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@39693e3b; decorates RemoteLauncher[hudson.remoting.Channel@562553fd:JNLP4-connect connection from 10.233.68.150/10.233.68.150:35000] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G06 Run cases: relay_interrupt safe_mode sequence_safe_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh Agent dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8v is provisioned from template dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a9d2e033c1e143b8977ffdd958b5369edb4911db" jenkins/label: "dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e" name: "dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8v" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8v" - name: "JENKINS_NAME" value: "dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8v" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8v in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] { [Pipeline] checkout Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" The recommended git tool is: git [Pipeline] withEnv [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] container [Pipeline] { Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) [Pipeline] sh > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" [Pipeline] withEnv [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [relay_interrupt] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetGlobalVariableFailed=return("server_uuid,1152") + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K Apr 26 11:12 . drwxr-sr-x 17 1000 1000 4.0K Apr 26 11:12 .. -rw------- 1 root 1000 1.7K Apr 26 11:12 ca-key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:12 ca.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:12 client-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:12 client-key.pem -rw------- 1 root 1000 1.7K Apr 26 11:12 private_key.pem -rw-r--r-- 1 root 1000 451 Apr 26 11:12 public_key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:12 server-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:12 server-key.pem [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] // container + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K Apr 26 11:12 . drwxr-sr-x 17 1000 1000 4.0K Apr 26 11:12 .. -rw------- 1 root 1000 1.7K Apr 26 11:12 ca-key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:12 ca.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:12 client-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:12 client-key.pem -rw------- 1 root 1000 1.7K Apr 26 11:12 private_key.pem -rw-r--r-- 1 root 1000 451 Apr 26 11:12 public_key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:12 server-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:12 server-key.pem [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache [Fri Apr 26 19:12:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@612cf8a8; decorates RemoteLauncher[hudson.remoting.Channel@4f1995:JNLP4-connect connection from 10.233.68.63/10.233.68.63:58594] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:12:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time Agent dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4-hvg39 is provisioned from template dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "aa1db85f4a3befc110699f8dfe5930cbef10d8a3" jenkins/label: "dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7" name: "dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4-hvg39" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4-hvg39" - name: "JENKINS_NAME" value: "dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4-hvg39" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-4730bf40-4a02-4a77-9996-53e143fb84f7-739f4-hvg39 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/0/source1.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 waiting for asynchronous relay and subtask to be started dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status" reset go failpoints, and need restart dm-worker then resume task, task will recover success wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:12:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully read binlog from relay log failed, and will use remote binlog wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:12:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetSessionVariableFailed=return("sql_mode,1152") Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 30.23 secs (104186905 bytes/sec) [Pipeline] { [Pipeline] } [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh Agent dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d is provisioned from template dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "92148bc531a13215a70c7c8f3609b26ae8948afa" jenkins/label: "dm-it-efa49597-2e43-42f3-a985-20ca698befaa" name: "dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d" - name: "JENKINS_NAME" value: "dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] cache + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:12 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:12 .. -rw-r--r-- 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 Apr 26 19:12 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:12 .git drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 .github -rw-r--r-- 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r-- 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r-- 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:12 bin drwxr-sr-x 17 jenkins jenkins 4.0K Apr 26 19:12 cdc drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:12 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K Apr 26 19:12 cmd drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 deployments drwxr-sr-x 26 jenkins jenkins 4.0K Apr 26 19:12 dm drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:12 docs drwxr-sr-x 13 jenkins jenkins 4.0K Apr 26 19:12 engine -rwxr-xr-x 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 examples -rw-r--r-- 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r-- 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 metrics drwxr-sr-x 45 jenkins jenkins 4.0K Apr 26 19:12 pkg drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:12 proto drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:12 scripts drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:12 tests -rw-r--r-- 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break Agent dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0-r94xj is provisioned from template dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "74f837653b67e0dd91e3779ac0fd44f0f92a4d1e" jenkins/label: "dm-it-e442b60e-8d6d-4c41-982d-924e09074d98" name: "dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0-r94xj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0-r94xj" - name: "JENKINS_NAME" value: "dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0-r94xj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-e442b60e-8d6d-4c41-982d-924e09074d98-vkxn0-r94xj in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Fri Apr 26 19:12:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:12:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/1/source1.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" waiting for asynchronous relay and subtask to be started dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status" reset go failpoints, and need restart dm-worker then resume task, task will recover success wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:13:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 9.74 secs (323460953 bytes/sec) [Pipeline] { [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout The recommended git tool is: git + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:13 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:13 .. -rw-r--r-- 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 Apr 26 19:12 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:12 .git drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:12 .github -rw-r--r-- 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r-- 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r-- 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:13 bin drwxr-sr-x 17 jenkins jenkins 4.0K Apr 26 19:13 cdc drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:13 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K Apr 26 19:13 cmd drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:13 deployments drwxr-sr-x 26 jenkins jenkins 4.0K Apr 26 19:13 dm drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:13 docs drwxr-sr-x 13 jenkins jenkins 4.0K Apr 26 19:13 engine -rwxr-xr-x 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:13 examples -rw-r--r-- 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r-- 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:13 metrics drwxr-sr-x 45 jenkins jenkins 4.0K Apr 26 19:13 pkg drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:13 proto drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:13 scripts drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:13 tests -rw-r--r-- 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:13 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] checkout [Pipeline] stage [Pipeline] { (Test) The recommended git tool is: git [Pipeline] { [Pipeline] // timeout [Pipeline] sh [Pipeline] } [Pipeline] checkout [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { The recommended git tool is: git [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1950f111; decorates RemoteLauncher[hudson.remoting.Channel@60d0108d:JNLP4-connect connection from 10.233.93.206/10.233.93.206:57888] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] // timeout [Pipeline] sh + '[' TLS_GROUP == G01 ']' + echo 'run G01 test' run G01 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN + make dm_integration_test_in_group GROUP=G01 [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@690f3220; decorates RemoteLauncher[hudson.remoting.Channel@2feb4302:JNLP4-connect connection from 10.233.123.3/10.233.123.3:38384] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh + '[' TLS_GROUP == G10 ']' + echo 'run G10 test' run G10 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + make dm_integration_test_in_group GROUP=G10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@57b9762a; decorates RemoteLauncher[hudson.remoting.Channel@ae9fc01:JNLP4-connect connection from 10.233.105.44/10.233.105.44:48682] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] container [Pipeline] { [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:13 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:13 .. -rw-------. 1 root 1000 1.7K Apr 26 11:13 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:13 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 server-key.pem [Pipeline] } [Pipeline] // container + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:13 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:13 .. -rw-------. 1 root 1000 1.7K Apr 26 11:13 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:13 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 server-key.pem [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } [Pipeline] cache Agent dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns-jfrv4 is provisioned from template dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "df92f59f1fe7ba768ea3e9203dee958c5acb907a" jenkins/label: "dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56" name: "dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns-jfrv4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns-jfrv4" - name: "JENKINS_NAME" value: "dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns-jfrv4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-91434a49-4c78-40b7-a63c-4ca3663e0b56-jtxns-jfrv4 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G01 Run cases: ha_cases3 ha_cases3_1 ha_master ... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G10 Run cases: start_task print_status http_apis new_relay all_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:13:10 CST 2024] <<<<<< start test_stop_task >>>>>> [Fri Apr 26 19:13:10 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [start_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:13:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Commit message: "feat(tidb): test flashbacktest package (#2942)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 import prepare data start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=1 expected=1 start DM worker and master [Fri Apr 26 19:13:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:13:13 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:13:13 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:13:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 check diff successfully read binlog from relay log failed, and will use remote binlog wait process dm-worker.test exit... Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:13:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time Commit message: "feat(tidb): test flashbacktest package (#2942)" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Fri Apr 26 19:13:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source1.yaml" [Fri Apr 26 19:13:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source2.yaml" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-04-26 13:13:18'" [Fri Apr 26 19:13:21 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker3.toml >>>>>> dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" [Fri Apr 26 19:13:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8264 is alive [Fri Apr 26 19:13:23 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker2" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-04-26 13:13:18'" rpc addr 127.0.0.1:18262 is alive [Fri Apr 26 19:13:24 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '1995-03-07 01:02:03'" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml " wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2037-12-12 01:02:03'" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase Agent dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2 is provisioned from template dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e9344e404c486c030a3a6c5507acb03f600a2acb" jenkins/label: "dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680" name: "dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2" - name: "JENKINS_NAME" value: "dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:13:29 CST 2024] <<<<<< test case relay_interrupt success! >>>>>> start running case: [safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:13:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 19.50 secs (161522850 bytes/sec) [Pipeline] { rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:13:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { wait process dm-worker.test exit... [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { wait process dm-worker.test exit... [Pipeline] cache rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:13:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit use sync_diff_inspector to check increment data check diff successfully check diff successfully [Fri Apr 26 19:13:32 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task.yaml --remove-meta" got=2 expected=2 start dumping SQLs into source stop tasks test dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "query-status test" got=1 expected=1 stop tasks test2 dmctl test cmd: "stop-task test2" got=3 expected=3 dmctl test cmd: "query-status test2" got=1 expected=1 [Fri Apr 26 19:13:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully Agent dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl is provisioned from template dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "11c55f942164a2728b3355592a5bbba9ce24843c" jenkins/label: "dm-it-3f0cadde-a643-4033-a34c-a2527e3151db" name: "dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl" - name: "JENKINS_NAME" value: "dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:13:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "list-member --worker" got=2 expected=2 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive Agent dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h is provisioned from template dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "71c300ff399bea4ed28260210886bc5843a123a1" jenkins/label: "dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1" name: "dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h" - name: "JENKINS_NAME" value: "dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/source1.yaml" dmctl test cmd: "query-status test" got=4 expected=4 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml" wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test2" got=4 expected=4 wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 1 [Fri Apr 26 19:13:42 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:13:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:13:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" check diff successfully check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:13:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 15.75 secs (200026700 bytes/sec) [Pipeline] { [Pipeline] sh wait process dm-master.test exit... process dm-master.test already exit [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:13 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:13 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:13 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:13 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:13 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:13 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:13 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:13 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:13 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:13 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:13 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:13 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:13 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:13 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:13 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:13:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:13 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:13 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:13 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:13 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:13 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:13 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:13 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:13 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:13 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:13 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:13 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:13 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:13 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:13 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:13 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:13 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] checkout The recommended git tool is: git wait process dm-worker.test exit... [Pipeline] checkout The recommended git tool is: git [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] // timeout The recommended git tool is: git [Pipeline] sh [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] // timeout [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@130db8fd; decorates RemoteLauncher[hudson.remoting.Channel@6a4d6557:JNLP4-connect connection from 10.233.68.14/10.233.68.14:35266] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + '[' TLS_GROUP == G07 ']' + echo 'run G07 test' run G07 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ wait process dm-worker.test exit... Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2fe34164; decorates RemoteLauncher[hudson.remoting.Channel@1c625be1:JNLP4-connect connection from 10.233.71.164/10.233.71.164:58254] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchTargetDoTablesFailed=return(1152) + make dm_integration_test_in_group GROUP=G07 + '[' TLS_GROUP == G03 ']' + echo 'run G03 test' run G03 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@17a7d232; decorates RemoteLauncher[hudson.remoting.Channel@42993704:JNLP4-connect connection from 10.233.72.33/10.233.72.33:36426] will be ignored (a typical symptom is the Git executable not being run inside a designated container) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@200d9c2d; decorates RemoteLauncher[hudson.remoting.Channel@233f1f35:JNLP4-connect connection from 10.233.72.184/10.233.72.184:42696] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 wait process dm-worker.test exit... process dm-worker.test already exit Cloning repository https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 + make dm_integration_test_in_group GROUP=G03 [Pipeline] { [Pipeline] { [Pipeline] { [Fri Apr 26 19:13:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> success success [Fri Apr 26 19:13:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:13:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] sh wait for rpc addr 127.0.0.1:8261 alive the 1-th time Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G07 Run cases: shardddl1 shardddl1_1 shardddl2 shardddl2_1 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K Apr 26 11:13 . drwxr-sr-x 17 1000 1000 4.0K Apr 26 11:13 .. -rw------- 1 root 1000 1.7K Apr 26 11:13 ca-key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:13 ca.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:13 client-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:13 client-key.pem -rw------- 1 root 1000 1.7K Apr 26 11:13 private_key.pem -rw-r--r-- 1 root 1000 451 Apr 26 11:13 public_key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:13 server-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:13 server-key.pem [Pipeline] sh > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:13 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:13 .. -rw-------. 1 root 1000 1.7K Apr 26 11:13 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:13 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:13 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:13 server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G03 Run cases: dmctl_advance dmctl_basic dmctl_command ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:13:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] } [Pipeline] } + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K Apr 26 11:13 . drwxr-sr-x 17 1000 1000 4.0K Apr 26 11:13 .. -rw------- 1 root 1000 1.7K Apr 26 11:13 ca-key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:13 ca.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:13 client-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:13 client-key.pem -rw------- 1 root 1000 1.7K Apr 26 11:13 private_key.pem -rw-r--r-- 1 root 1000 451 Apr 26 11:13 public_key.pem -rw-r--r-- 1 root 1000 1.1K Apr 26 11:13 server-cert.pem -rw------- 1 root 1000 1.7K Apr 26 11:13 server-key.pem rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 [Pipeline] // container [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } [Pipeline] cache Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) Commit message: "feat(tidb): test flashbacktest package (#2942)" Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/0/source1.yaml" check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" start task and will failed > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 Commit message: "feat(tidb): test flashbacktest package (#2942)" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [shardddl1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:13:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time version() 5.7.26-log version() 8.0.21 start running case: [dmctl_advance] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm [Fri Apr 26 19:13:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock unlock" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) dmctl test cmd: "shard-ddl-lock unlock lock_id --force-remove=invalid-force-remove" dmctl test cmd: "handle-error" Commit message: "feat(tidb): test flashbacktest package (#2942)" dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner dmctl test cmd: "handle-error test-task --binlog-pos mysql-bin:shoud-bin-digital skip" dmctl test cmd: "handle-error test-task replace alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "handle-error test-task wrong_operation" dmctl test cmd: "binlog" dmctl test cmd: "binlog skip test-task --binlog-pos mysql-bin:should-be-digital" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:13:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "binlog replace test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog inject test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog list test-task --binlog-pos mysql-bin:should-be-digital" dmctl test cmd: "binlog wrong_operation test-task" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 dmctl test cmd: "binlog-schema" dmctl test cmd: "binlog-schema test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source1.yaml" dmctl test cmd: "binlog-schema list test" dmctl test cmd: "binlog-schema update test" wait process dm-master.test exit... [Fri Apr 26 19:14:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> dmctl test cmd: "binlog-schema update test test_db test_table --from-source" dmctl test cmd: "binlog-schema update test test_db test_table --from-source --from-target" wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog-schema update test test_db test_table schema.sql --from-source" dmctl test cmd: "binlog-schema delete test" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:14:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source2.yaml" wait process dm-master.test exit... check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success success finish running run safe mode recover case 1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:14:03 CST 2024] <<<<<< test case dmctl_advance success! >>>>>> start running case: [dmctl_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm dmctl test cmd: "any command" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 5.22 secs (603913426 bytes/sec) [Pipeline] { [Pipeline] cache wait process dm-master.test exit... wait process dm-worker exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:14:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker exit... process dm-worker already exit [Fri Apr 26 19:14:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:14:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> Agent dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd-tzt6q is provisioned from template dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/272/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e0c9d15825f3c33b181930a8410da85af92a329b" jenkins/label: "dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b" name: "dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd-tzt6q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd-tzt6q" - name: "JENKINS_NAME" value: "dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd-tzt6q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" wait for rpc addr 127.0.0.1:8262 alive the 1-th time Running on dm-it-6e1849a9-6be5-49d7-a309-ac6d431fb20b-sbssd-tzt6q in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test rpc addr 127.0.0.1:8262 is alive wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:14:06 CST 2024] <<<<<< start DM-COMPACT >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "any command" [Fri Apr 26 19:14:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test" got=1 expected=1 wait for rpc addr 127.0.0.1:8261 alive the 2-th time start to run safe mode case 2 [Fri Apr 26 19:14:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:14:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> check_task_wrong_arg dmctl test cmd: "check-task" dmctl test cmd: "check-task not_exists_config_file" pause_relay_wrong_arg dmctl test cmd: "pause-relay wrong_arg" wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "pause-relay" resume_relay_wrong_arg dmctl test cmd: "resume-relay wrong_arg" dmctl test cmd: "resume-relay" pause_task_wrong_arg dmctl test cmd: "pause-task" dmctl test cmd: "pause-task -s a -s b" resume_task_wrong_arg dmctl test cmd: "resume-task" dmctl test cmd: "resume-task -s a -s b" query_status_wrong_arg dmctl test cmd: "query-status wrong_args_count more_than_one" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:14:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status -s source-x task-y" start_task_wrong_arg dmctl test cmd: "start-task" dmctl test cmd: "start-task not_exists_config_file" stop_task_wrong_arg dmctl test cmd: "stop-task" dmctl test cmd: "stop-task -s a -s b" show_ddl_locks_wrong_arg dmctl test cmd: "shard-ddl-lock a b" dmctl test cmd: "show-ddl-locks a b" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " purge_relay_wrong_arg dmctl test cmd: "purge-relay wrong_arg" dmctl test cmd: "purge-relay" dmctl test cmd: "purge-relay --filename bin-000001 -s mysql-replica-01 -s mysql-replica-02" operate_source_empty_arg dmctl test cmd: "operate-source" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "operate-source create not_exists_config_file" dmctl test cmd: "operate-source invalid /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" dmctl test cmd: "operate-source stop /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" transfer_source_empty_arg dmctl test cmd: "transfer-source" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "transfer-source source-id" dmctl test cmd: "transfer-source source-id worker-id another-worker" start_relay_empty_arg dmctl test cmd: "start-relay" dmctl test cmd: "start-relay wrong_arg" stop_relay_empty_arg dmctl test cmd: "stop-relay" dmctl test cmd: "stop-relay wrong_arg" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" source-id: mysql-replica-01 server-id: 123456 enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' enable-relay: false from: host: 127.0.0.1 user: root port: 3306 filters: - schema-pattern: "dmctl" table-pattern: "t_1" sql-pattern: ["alter table .* add column aaa int"] action: Ignore - schema-pattern: "dmctl" table-pattern: "t_2" sql-pattern: ["alter table .* add column aaa int"] action: Ignore dmctl test cmd: "operate-source stop /tmp/dm_test/dmctl_basic/source1-wrong.yaml mysql-replica-02" wait process dm-master.test exit... check diff failed 1-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" [Fri Apr 26 19:14:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit check diff successfully rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" dmctl test cmd: "operate-source show" [Fri Apr 26 19:14:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 2-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" dmctl test cmd: "operate-source show -s mysql-replica-01" ############################################################ test case for operate tasks bound to a source ############################################################ dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/foo-task.yaml" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:14:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... check diff failed 3-th time, retry later rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:14:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchAllDoTablesFailed=return(1152) dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" [Fri Apr 26 19:14:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "pause-task -s xxx" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "resume-task -s xxx" dmctl test cmd: "stop-task -s xxx" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "pause-task -s mysql-replica-01" check diff failed 4-th time, retry later dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:14:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again wait process dm-worker.test exit... process dm-worker.test already exit success success [Fri Apr 26 19:14:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:14:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 5-th time, retry later dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" wait process dm-worker.test exit... process dm-worker.test already exit check diff failed 6-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:14:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 19.18 secs (164220237 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Fri Apr 26 19:14:25 CST 2024] <<<<<< finish DM-COMPACT >>>>>> [Pipeline] { [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" wait process dm-worker exit... [Pipeline] sh [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:14 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:14 .. -rw-r--r-- 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 Apr 26 19:13 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:13 .git drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:13 .github -rw-r--r-- 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r-- 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r-- 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:14 bin drwxr-sr-x 17 jenkins jenkins 4.0K Apr 26 19:14 cdc drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:14 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K Apr 26 19:14 cmd drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 deployments drwxr-sr-x 26 jenkins jenkins 4.0K Apr 26 19:14 dm drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:14 docs drwxr-sr-x 13 jenkins jenkins 4.0K Apr 26 19:14 engine -rwxr-xr-x 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 examples -rw-r--r-- 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r-- 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 metrics drwxr-sr-x 45 jenkins jenkins 4.0K Apr 26 19:14 pkg drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:14 proto drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:14 scripts drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:14 tests -rw-r--r-- 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:14 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:14 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:14 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:14 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:14 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:14 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:14 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:14 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:14 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:14 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:14 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:14 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:14 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:14 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:14 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:14 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:14 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:14 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:14 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:14 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker exit... process dm-worker already exit [Fri Apr 26 19:14:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:14:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:14:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/baz-task.yaml" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:14:28 CST 2024] <<<<<< start DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success success finish running run safe mode recover case 2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive [Pipeline] cache dmctl test cmd: "pause-task -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-master.test exit... run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" wait process dm-worker.test exit... run tidb sql failed 2-th time, retry later dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-worker.test exit... dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 3 [Fri Apr 26 19:14:33 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time run tidb sql failed 3-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "transfer-source mysql-replica-01 worker1" dmctl test cmd: "transfer-source mysql-replica-01 worker2" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:14:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive kill worker2 wait process dm-master.test exit... wait process worker2 exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:14:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 4-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process worker2 exit... process worker2 already exit dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" restart worker2 [Fri Apr 26 19:14:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" run tidb sql failed 5-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-worker.test exit... dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-01 worker2" check diff successfully dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "stop-relay -s mysql-replica-01" pause_relay_success dmctl test cmd: "pause-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:14:39 CST 2024] <<<<<< start test_COMMIT_in_QueryEvent >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit got=3 expected=3 got=2 expected=2 dmctl test cmd: "resume-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" [Fri Apr 26 19:14:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/start_task/1/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time run tidb sql failed 6-th time, retry later dmctl_check_task dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task2.yaml" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:14:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/start_task/1/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 100 -w 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 0" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:14:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" run tidb sql failed 7-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" check_task_optimistic dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task4.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" check_task_only_warning dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/start_task/1/dm-task.yaml --remove-meta=true" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:14:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 8-th time, retry later dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" check task with empty unit config dmctl test cmd: "check-task /tmp/empty-dump.yaml" --- name: empty-unit-task task-mode: all target-database: host: "127.0.0.1" port: 4000 user: "root" password: "" mysql-instances: - source-id: "mysql-replica-01" mydumper-config-name: "global" loader-config-name: "global" syncer-config-name: "global" block-allow-list: "instance" mydumpers: global: threads: 4 loaders: global: syncers: global: worker-count: 32 block-allow-list: instance: do-dbs: ["dmctl"] do-tables: - db-name: "dmctl" tbl-name: "~^t_[\\d]+"dmctl test cmd: "check-task /tmp/empty-load.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "check-task /tmp/empty-sync.yaml" start task with empty unit config dmctl test cmd: "start-task /tmp/empty-cfg.yaml" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff failed 1-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "config task empty-unit-task" dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/empty-unit-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/dmctl_basic/dm-task-error-database-config.yaml" dmctl_start_task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" run tidb sql failed 9-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit success success [Fri Apr 26 19:14:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:14:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully <<<<<< test_COMMIT_in_QueryEvent success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test_incremental_no_source_meta" got=3 expected=3 dmctl test cmd: "stop-task test_incremental_no_source_meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml " rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" wait process dm-master.test exit... got=3 expected=3 run tidb sql failed 10-th time, retry later check diff successfully dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" wait process dm-master.test exit... process dm-master.test already exit Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 18.02 secs (174801388 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { wait process dm-worker.test exit... run tidb sql failed 11-th time, retry later [Pipeline] } [Pipeline] { wait process dmctl_basic/worker1 exit... wait process dm-worker.test exit... + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:14 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 26 19:14 .. -rw-r--r-- 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 Apr 26 19:14 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:14 .git drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 .github -rw-r--r-- 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r-- 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r-- 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:14 bin drwxr-sr-x 17 jenkins jenkins 4.0K Apr 26 19:14 cdc drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:14 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K Apr 26 19:14 cmd drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 deployments drwxr-sr-x 26 jenkins jenkins 4.0K Apr 26 19:14 dm drwxr-sr-x 7 jenkins jenkins 4.0K Apr 26 19:14 docs drwxr-sr-x 13 jenkins jenkins 4.0K Apr 26 19:14 engine -rwxr-xr-x 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 examples -rw-r--r-- 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r-- 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 metrics drwxr-sr-x 45 jenkins jenkins 4.0K Apr 26 19:14 pkg drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:14 proto drwxr-sr-x 3 jenkins jenkins 4.0K Apr 26 19:14 scripts drwxr-sr-x 5 jenkins jenkins 4.0K Apr 26 19:14 tests -rw-r--r-- 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K Apr 26 19:14 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:14:52 CST 2024] <<<<<< test case start_task success! >>>>>> start running case: [print_status] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh... Verbose mode = false +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status ++ pwd + cur=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../_utils/test_prepare ++ source env_variables +++ MYSQL_HOST1=127.0.0.1 +++ MYSQL_HOST2=127.0.0.1 +++ TIDB_HOST=127.0.0.1 +++ MYSQL_PORT1=3306 +++ MYSQL_PORT2=3307 +++ MYSQL_PASSWORD1=123456 +++ MYSQL_PASSWORD2=123456 +++ TIDB_PASSWORD=123456 +++ TIDB_PORT=4000 +++ MASTER_PORT=8261 +++ MASTER_PEER_PORT=8291 +++ MASTER_PORT1=8261 +++ MASTER_PORT2=8361 +++ MASTER_PORT3=8461 +++ MASTER_PORT4=8561 +++ MASTER_PORT5=8661 +++ MASTER_PORT6=8761 +++ MASTER_PEER_PORT1=8291 +++ MASTER_PEER_PORT2=8292 +++ MASTER_PEER_PORT3=8293 +++ MASTER_PEER_PORT4=8294 +++ MASTER_PEER_PORT5=8295 +++ MASTER_PEER_PORT6=8296 +++ WORKER1_PORT=8262 +++ WORKER2_PORT=8263 +++ WORKER3_PORT=8264 +++ WORKER4_PORT=18262 +++ WORKER5_PORT=18263 +++ WORKER1_NAME=worker1 +++ WORKER2_NAME=worker2 +++ SOURCE_ID1=mysql-replica-01 +++ SOURCE_ID2=mysql-replica-02 +++ RESET_MASTER=true +++ VERBOSE=false ++ '[' true = true ']' ++ run_sql 'RESET MASTER' 3306 123456 ++ run_sql 'RESET MASTER' 3307 123456 + WORK_DIR=/tmp/dm_test/print_status + cleanup_data print_status + rm -rf /tmp/dm_test/print_status + mkdir /tmp/dm_test/print_status + for target_db in '"$@"' + run_sql 'drop database if exists `print_status`' 4000 123456 + run_sql 'drop database if exists dm_meta' 4000 123456 + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=0 + echo '0 dm-master alive' 0 dm-master alive + pkill -hup dm-master.test + true ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=0 + echo '0 dm-worker alive' 0 dm-worker alive + pkill -hup dm-worker.test + true ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test process dm-master.test already exit + wait_process_exit dm-worker.test process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + run + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.prepare.sql 127.0.0.1 3306 123456 wait process dmctl_basic/worker1 exit... process dmctl_basic/worker1 already exit run tidb sql failed 12-th time, retry later + inject_points=("github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)" "github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)" "github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)") ++ join_string ';' 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)' 'github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)' 'github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' ++ local 'IFS=;' ++ shift ++ echo 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + GO_FAILPOINTS='github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-worker1.toml /tmp/dm_test/print_status/dm-worker1.toml + sed -i s%placeholder%/tmp/dm_test/print_status/relay_by_worker%g /tmp/dm_test/print_status/dm-worker1.toml + run_dm_master /tmp/dm_test/print_status/master 8261 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml [Fri Apr 26 19:14:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_master_online 127.0.0.1:8261 wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dmctl_basic/worker2 exit... [Pipeline] } [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) rpc addr 127.0.0.1:8261 is alive + run_dm_worker /tmp/dm_test/print_status/worker1 8262 /tmp/dm_test/print_status/dm-worker1.toml [Fri Apr 26 19:14:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/print_status/dm-worker1.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_worker_online 127.0.0.1:8262 wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] // timeout [Pipeline] sh wait process dmctl_basic/worker2 exit... process dmctl_basic/worker2 already exit dmctl test cmd: "query-status" + '[' TLS_GROUP == TLS_GROUP ']' + echo 'run tls test' run tls test + echo 'copy mysql certs' copy mysql certs + sudo mkdir -p /var/lib/mysql + sudo chmod 777 /var/lib/mysql + sudo chown -R 1000:1000 /var/lib/mysql + sudo cp -r /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ca-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ca.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/client-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/client-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/private_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/public_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/server-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/server-key.pem /var/lib/mysql/ + sudo chown -R 1000:1000 /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem + ls -alh /var/lib/mysql/ total 40K drwxrwxrwx 2 jenkins jenkins 4.0K Apr 26 19:14 . drwxr-xr-x 1 root root 4.0K Apr 26 19:14 .. -rw------- 1 jenkins jenkins 1.7K Apr 26 19:14 ca-key.pem -rw-r--r-- 1 jenkins jenkins 1.1K Apr 26 19:14 ca.pem -rw-r--r-- 1 jenkins jenkins 1.1K Apr 26 19:14 client-cert.pem -rw------- 1 jenkins jenkins 1.7K Apr 26 19:14 client-key.pem -rw------- 1 jenkins jenkins 1.7K Apr 26 19:14 private_key.pem -rw-r--r-- 1 jenkins jenkins 451 Apr 26 19:14 public_key.pem -rw-r--r-- 1 jenkins jenkins 1.1K Apr 26 19:14 server-cert.pem -rw------- 1 jenkins jenkins 1.7K Apr 26 19:14 server-key.pem + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] // timeout [Pipeline] sh run tidb sql failed 13-th time, retry later + make dm_integration_test_in_group GROUP=TLS_GROUP got=2 expected=2 got=2 expected=2 [Fri Apr 26 19:14:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@c2f2396; decorates RemoteLauncher[hudson.remoting.Channel@4c3eea38:JNLP4-connect connection from 10.233.70.24/10.233.70.24:52192] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] } Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + '[' TLS_GROUP == G04 ']' + echo 'run G04 test' run G04 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ rpc addr 127.0.0.1:8262 is alive + cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/source1.yaml /tmp/dm_test/print_status/source1.yaml + sed -i '/relay-binlog-name/i\relay-dir: /tmp/dm_test/print_status/worker1/relay_log' /tmp/dm_test/print_status/source1.yaml + dmctl_operate_source create /tmp/dm_test/print_status/source1.yaml mysql-replica-01 + operate=create + source_toml=/tmp/dm_test/print_status/source1.yaml + source_id=mysql-replica-01 + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'operate-source create /tmp/dm_test/print_status/source1.yaml' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "operate-source create /tmp/dm_test/print_status/source1.yaml" check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success [Pipeline] // timeout [Pipeline] sh + make dm_integration_test_in_group GROUP=G04 success finish running run safe mode recover case 3 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:14:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time + '[' TLS_GROUP == G05 ']' + echo 'run G05 test' run G05 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] { [Pipeline] { Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh TLS_GROUP Run cases: tls ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container + make dm_integration_test_in_group GROUP=G05 [Pipeline] { + dmctl_start_task_standalone + '[' 0 -ge 2 ']' + remove_meta= + '[' 0 -ge 1 ']' + task_conf=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml ' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml " [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G04 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Run cases: import_goroutine_leak incremental_mode initial_unit import_goroutine_leak not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-master.test exit... [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:14 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:14 .. -rw-------. 1 root 1000 1.7K Apr 26 11:14 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:14 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 server-key.pem [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:14 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:14 .. -rw-------. 1 root 1000 1.7K Apr 26 11:14 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:14 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 server-key.pem rpc addr 127.0.0.1:8263 is alive config dmctl test cmd: "config" dmctl test cmd: "config hihi haha" dmctl test cmd: "config task haha" dmctl test cmd: "config master haha" [Pipeline] sh run tidb sql failed 14-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G05 Run cases: load_interrupt many_tables online_ddl load_interrupt not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "config worker haha" dmctl test cmd: "config source haha" + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:14 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:14 .. -rw-------. 1 root 1000 1.7K Apr 26 11:14 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:14 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 server-key.pem + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check diff successfully + ls /tmp/dm_test/print_status/relay_by_worker/worker1/ceac036c-03bd-11ef-b7d4-82ae00ed5d93.000001 /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/ceac036c-03bd-11ef-b7d4-82ae00ed5d93.000001: dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk-bin.000001 relay.meta + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.increment.sql 127.0.0.1 3306 123456 + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check diff failed 1-th time, retry later Avoid second fetch Checking out Revision cb9bc9e8822a01a5d59a2f670fb429e588065145 (origin/main) dmctl test cmd: "get-config haha" dmctl test cmd: "get-config haha hihi" + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:14 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:14 .. -rw-------. 1 root 1000 1.7K Apr 26 11:14 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:14 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:14 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:14 server-key.pem Commit message: "feat(tidb): test flashbacktest package (#2942)" dmctl test cmd: "get-config master haha" dmctl test cmd: "get-config worker haha" [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } dmctl test cmd: "get-config source haha" dmctl test cmd: "get-config task haha" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f cb9bc9e8822a01a5d59a2f670fb429e588065145 # timeout=10 [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow dmctl test cmd: "get-task-config haha" dmctl test cmd: "get-task-config haha hihi" [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [tls] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Pipeline] cache wait process dm-worker.test exit... dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [incremental_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:14:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time run tidb sql failed 15-th time, retry later got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "stop-task test" wait process dm-worker.test exit... dmctl test cmd: "operate-source stop mysql-replica-01" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [many_tables] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:14:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully + check_log_contains /tmp/dm_test/print_status/worker1/log/dm-worker.log 'enable safe-mode because of task initialization.*duration=1m0s' + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=1 + echo '1 dm-master alive' 1 dm-master alive + pkill -hup dm-master.test ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=1 + echo '1 dm-worker alive' 1 dm-worker alive + pkill -hup dm-worker.test ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source1.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 4 [Fri Apr 26 19:15:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:15:00 CST 2024] <<<<<< finish DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "operate-source stop mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source2.yaml" wait process dm-master.test exit... process dm-master.test already exit + wait_process_exit dm-worker.test wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8263 is alive wait process dm-worker exit... dmctl test cmd: "start-task /tmp/dm_test/dmctl_basic/get_task.yaml" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:15:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker exit... process dm-worker already exit [Fri Apr 26 19:15:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:15:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:15:03 CST 2024] <<<<<< start DM-MULTIPLE_ROWS >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" start prepare_data [Fri Apr 26 19:15:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Fri Apr 26 19:15:03 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Fri Apr 26 19:15:03 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + check_print_status + i=0 + '[' 0 -lt 3 ']' ++ grep 'dm-worker exit' /tmp/dm_test/print_status/worker1/log/dm-worker.log + exit_log='[2024/04/26 19:15:02.071 +08:00] [INFO] [main.go:93] ["dm-worker exit"]' + '[' '[2024/04/26 19:15:02.071 +08:00] [INFO] [main.go:93] ["dm-worker exit"]' == 'not found' ']' + break + '[' 0 -ge 3 ']' + echo 'checking print status' checking print status + dump_status_file=/tmp/dm_test/print_status/worker1/log/dump_status.log + grep -o 'progress status of dumpling' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=1 + '[' 1 -ge 1 ']' + grep -o '\[bps=0' /tmp/dm_test/print_status/worker1/log/dm-worker.log + true ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=0 + '[' 0 -eq 0 ']' + status_file=/tmp/dm_test/print_status/worker1/log/loader_status.log + grep -oP '\[unit=lightning-load\] \[IsCanceled=false\] \[finished_bytes=59674\] \[total_bytes=59674\] \[progress=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/loader_status.log ++ awk '{print $1}' + status_count=1 + '[' 1 -eq 1 ']' + grep 'current speed (bytes / seconds)' /tmp/dm_test/print_status/worker1/log/dm-worker.log + grep -vq '"current speed (bytes / seconds)"=0' + echo 'check load unit print status success' check load unit print status success + status_file2=/tmp/dm_test/print_status/worker1/log/syncer_status.log + grep -oP '\[total_rows=[0-9]+\] \[total_rps=[0-9]+\] \[rps=[0-9]+\] \[master_position=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/syncer_status.log ++ awk '{print $1}' + status_count2=7 + '[' 7 -ge 1 ']' + echo 'check sync unit print status success' check sync unit print status success ++ date + echo '[Fri Apr 26 19:15:04 CST 2024] <<<<<< test case print_status success! >>>>>>' [Fri Apr 26 19:15:04 CST 2024] <<<<<< test case print_status success! >>>>>> start running case: [http_apis] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" wait process dm-master exit... dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "list-member" [Fri Apr 26 19:15:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Fri Apr 26 19:15:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time process dm-master already exit [Fri Apr 26 19:15:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8461 is alive [Fri Apr 26 19:15:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" check log contain failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" check diff successfully rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:15:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive check log contain failed 2-th time, retry later wait process dm-worker.test exit... check master alive dmctl test cmd: "list-member" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8262 is alive source-id: mysql-replica-01\nflavor: ''\nenable-gtid: true\nenable-relay: true\nrelay-dir: /tmp/dm_test/http_apis/worker1/relay_log\nrelay-binlog-name: ''\nrelay-binlog-gtid: ''\nfrom:\n host: 127.0.0.1\n user: root\n password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=\n port: 3306\n % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 470 100 166 100 304 1236 2263 --:--:-- --:--:-- --:--:-- 2268 start task and check stage ---\nname: test\ntask-mode: all\nis-sharding: false\nmeta-schema: \"dm_meta\"\n# enable-heartbeat: true\n\ntarget-database:\n host: \"127.0.0.1\"\n port: 4000\n user: \"root\"\n password: \"\"\n\nmysql-instances:\n - source-id: \"mysql-replica-01\"\n block-allow-list: \"instance\"\n mydumper-config-name: \"global\"\n loader-config-name: \"global\"\n syncer-config-name: \"global\"\n\nblock-allow-list:\n instance:\n do-dbs: [\"http_apis\"]\n\nmydumpers:\n global:\n threads: 4\n chunk-filesize: 0\n skip-tz-utc: true\n statement-size: 100\n extra-args: \"\"\n\nloaders:\n global:\n pool-size: 16\n dir: \"./dumped_data\"\n\nsyncers:\n global:\n worker-count: 16\n batch: 100\n HTTP 127.0.0.1:8261/apis/v1alpha1/status/test-task is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed wait process worker1 exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:15:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 742 0 0 100 742 0 740 0:00:01 0:00:01 --:--:-- 740 100 950 100 208 100 742 191 683 0:00:01 0:00:01 --:--:-- 683 100 950 100 208 100 742 191 683 0:00:01 0:00:01 --:--:-- 683 wait process worker1 exit... dmctl test cmd: "query-status test" run tidb sql failed 1-th time, retry later test http and api interface Release Version: v7.5.1-57-g3ca0c6ff1 Git Commit Hash: 3ca0c6ff17a09c4824dcb351353e4007cb572d5c Git Branch: HEAD UTC Build Time: 2024-04-26 11:01:06 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true rpc addr 127.0.0.1:8261 is alive wait process worker1 exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 { "result": true, "msg": "", "members": [ { "leader": { "msg": "", "name": "master2", "addr": "127.0.0.1:8361" } }, { "master": { "msg": "", "masters": [ { "name": "master1", "memberID": "10068700568285020701", "alive": true, "peerURLs": [ "https://127.0.0.1:8291" ], "clientURLs": [ "https://127.0.0.1:8261" ] }, { "name": "master2", "memberID": "518295880577331676", "alive": true, "peerURLs": [ "https://127.0.0.1:8292" ], "clientURLs": [ "https://127.0.0.1:8361" ] }, { "name": "master3", "memberID": "6800138986734450252", "alive": true, "peerURLs": [ "https://127.0.0.1:8293" ], "clientURLs": [ "https://127.0.0.1:8461" ] } ] } }, { "worker": { "msg": "", "workers": [ { "name": "worker1", "addr": "127.0.0.1:8262", "stage": "bound", "source": "mysql-replica-01" } ] } } ] } rpc addr 127.0.0.1:8261 is alive use common name not in 'cert-allowed-cn' should not request success wait for rpc addr 127.0.0.1:8261 alive the 1-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 265k 0 --:--:-- --:--:-- --:--:-- 301k get sub task configs % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2299 0 2299 0 0 444k 0 --:--:-- --:--:-- --:--:-- 561k pause task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 194 100 183 100 11 13979 840 --:--:-- --:--:-- --:--:-- 14076 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1960 100 1960 0 0 272k 0 --:--:-- --:--:-- --:--:-- 319k resume task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 195 100 184 100 11 8323 497 --:--:-- --:--:-- --:--:-- 8363 wait process worker1 exit... wait process worker1 exit... run tidb sql failed 2-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 3-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 295k 0 --:--:-- --:--:-- --:--:-- 301k wait process worker1 exit... wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 4-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 801 100 801 0 0 124k 0 --:--:-- --:--:-- --:--:-- 130k wait process worker1 exit... run tidb sql failed 3-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit success success [Fri Apr 26 19:15:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:15:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time kill dm-worker1 wait for rpc addr 127.0.0.1:8261 alive the 5-th time wait process worker1 exit... use sync_diff_inspector to check increment data check diff successfully check diff successfully [Fri Apr 26 19:15:03 CST 2024] <<<<<< finish test_stop_task >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" wait process dm-worker1 exit... wait for rpc addr 127.0.0.1:8261 alive the 6-th time dmctl test cmd: "list-member" [Fri Apr 26 19:15:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... got=3 expected=3 wait process worker1 exit... run tidb sql failed 4-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --worker --name=worker1" wait for rpc addr 127.0.0.1:8261 alive the 7-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... got=1 expected=1 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 6931 0 --:--:-- --:--:-- --:--:-- 8250 check data check diff successfully 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process worker1 exit... wait for rpc addr 127.0.0.1:8261 alive the 8-th time rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... wait process dm-master.test exit... run tidb sql failed 5-th time, retry later wait process worker1 exit... process worker1 already exit wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:15:18 CST 2024] <<<<<< test case http_apis success! >>>>>> start running case: [new_relay] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:15:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner wait process worker2 exit... wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source1.yaml" wait process dm-master.test exit... dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" check diff successfully 34 11 12 54 dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 9-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:15:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... process worker2 already exit dmctl test cmd: "list-member --worker" dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" got=2 expected=2 [Fri Apr 26 19:15:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:15:20 CST 2024] <<<<<< finish DM-MULTIPLE_ROWS >>>>>> dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 10-th time dmctl test cmd: "start-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" got=2 expected=2 start task in incremental mode with zero gtid/pos [Fri Apr 26 19:15:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 20.44 secs (154084084 bytes/sec) [Pipeline] { [Pipeline] cache rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker exit... wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 11-th time finish prepare_data [Fri Apr 26 19:15:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success success finish running run safe mode recover case 4 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker exit... process dm-worker already exit [Fri Apr 26 19:15:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:15:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 12-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:15:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "query-status test" show_ddl_locks_no_locks dmctl test cmd: "shard-ddl-lock test" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:15:23 CST 2024] <<<<<< start DM-CAUSALITY >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 13-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/many_tables/source1.yaml" dmctl test cmd: "show-ddl-locks test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status test" dmctl test cmd: "query-status" dmctl test cmd: "pause-task test" got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 14-th time got=3 expected=3 got=2 expected=2 dmctl master_addr dmctl test cmd: "list-member" dmctl test cmd: "list-member" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task.yaml " wait process dm-worker.test exit... wait process dm-worker.test exit... [Fri Apr 26 19:15:25 CST 2024] <<<<<< finish DM-CAUSALITY >>>>>> [Fri Apr 26 19:15:25 CST 2024] <<<<<< start DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 15-th time check log contain failed 2-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:15:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 wait for rpc addr 127.0.0.1:8261 alive the 16-th time dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later check log contain failed 3-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:15:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 17-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:15:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 18-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:15:29 CST 2024] <<<<<< finish DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> [Fri Apr 26 19:15:29 CST 2024] <<<<<< start DM-UpdateBARule optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" [Fri Apr 26 19:15:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 4-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 5.77 secs (545964375 bytes/sec) [Pipeline] { [Pipeline] cache wait for rpc addr 127.0.0.1:8261 alive the 19-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 20-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "list-member" got=3 expected=3 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_leak passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 check alive timeout check data dmctl test cmd: "start-task /tmp/dm_test/shardddl1/task.yaml" 0 in Load unit "progress": "0.00 %", dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task-safe-mode-duration.yaml --remove-meta" wait process dm-master.test exit... [2024/04/26 19:15:23.759 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=ec2ecb42-03bd-11ef-83f1-c2ee5499590a.000001/dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d-bin.000001] [2024/04/26 19:15:23.778 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=ec2ecb42-03bd-11ef-83f1-c2ee5499590a.000001/dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d-bin.000001] [2024/04/26 19:15:28.409 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=ec2ecb42-03bd-11ef-83f1-c2ee5499590a.000001/dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d-bin.000001] [2024/04/26 19:15:28.426 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=ec2ecb42-03bd-11ef-83f1-c2ee5499590a.000001/dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d-bin.000001] [2024/04/26 19:15:28.434 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=ec2ecb42-03bd-11ef-83f1-c2ee5499590a.000001/dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d-bin.000001] [2024/04/26 19:15:28.455 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=ec2ecb42-03bd-11ef-83f1-c2ee5499590a.000001/dm-it-efa49597-2e43-42f3-a985-20ca698befaa-h59j8-qkz4d-bin.000001] dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "pause-task test" check diff successfully ============================== test_master_ha_when_enable_tidb_and_only_ca_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "show-ddl-locks" got=1 expected=1 restart dm-master check diff successfully wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master exit... process dm-master already exit wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:15:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:15:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:15:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=3 expected=3 dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " wait process dm-master.test exit... 0 in Load unit "progress": "0.00 %", wait process dm-master.test exit... [Fri Apr 26 19:15:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:15:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "list-member" dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:15:40 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "show-ddl-locks" got=1 expected=1 check diff successfully finish running safe mode duration case 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "resume-task test" run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" wait process dm-master.test exit... wait process dm-master.test exit... got=3 expected=3 got=4 expected=4 check diff successfully dmctl test cmd: "pause-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "resume-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" wait process dm-master.test exit... check diff successfully dmctl test cmd: "purge-relay --filename dm-it-fda5e1e8-c443-4c60-a8cf-a2b891f5e69e-d8359-g5v8v-bin.000003 -s mysql-replica-01" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1 worker2" dmctl test cmd: "query-status test" got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "show-ddl-locks" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... 0 in Load unit "progress": "24.43 %", got=3 expected=3 dmctl test cmd: "transfer-source mysql-replica-01 worker1" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... [Fri Apr 26 19:15:42 CST 2024] <<<<<< finish DM-UpdateBARule optimistic >>>>>> [Fri Apr 26 19:15:42 CST 2024] <<<<<< start DM-RENAME_TABLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff successfully wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:15:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:15:46 CST 2024] <<<<<< test case dmctl_basic success! >>>>>> start running case: [dmctl_command] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh... Verbose mode = false wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:15:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully kill dm-worker1 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:15:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:15:47 CST 2024] <<<<<< finish DM-RENAME_TABLE pessimistic >>>>>> [Fri Apr 26 19:15:47 CST 2024] <<<<<< start DM-RENAME_TABLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:15:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait process dm-worker1 exit... 0 in Load unit "progress": "41.47 %", wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive exit code should be not zero exit code should be not zero [Fri Apr 26 19:15:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:15:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Fri Apr 26 19:15:48 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Fri Apr 26 19:15:48 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive check diff successfully [Fri Apr 26 19:15:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:15:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:15:49 CST 2024] <<<<<< finish DM-RENAME_TABLE optimistic >>>>>> [Fri Apr 26 19:15:49 CST 2024] <<<<<< start DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Fri Apr 26 19:15:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "query-status -s mysql-replica-01" got=3 expected=3 got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/wrong-source.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml -w worker1" dmctl test cmd: "list-member --name worker1" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w wrong-worker" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w worker1" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-only-ca.yaml" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "purge-relay --filename dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk-bin.000005 -s mysql-replica-01" kill dm-worker1 wait process dm-master.test exit... dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" wait process dm-worker1 exit... dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml" check diff successfully wait for port 8262 offline the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" 0 in Load unit "progress": "67.13 %", wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" wait for port 8262 offline the 2-th time wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 20.84 secs (151142186 bytes/sec) [Pipeline] { [Pipeline] cache check diff successfully check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "resume-task test" got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-2.yaml --remove-meta=true" check diff successfully dmctl test cmd: "query-status test" [Fri Apr 26 19:15:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:15:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" got=3 expected=3 check diff failed 1-th time, retry later [Fri Apr 26 19:15:54 CST 2024] <<<<<< finish DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-master exit... { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "query-status test" dmctl test cmd: "query-status test2" check data check diff successfully check diff successfully ============================== test_worker_handle_multi_tls_tasks success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit [Fri Apr 26 19:15:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... check diff successfully dmctl test cmd: "config export -p /tmp/configs" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... 0 in Load unit "progress": "92.78 %", wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 3648 0 3648 0 0 464k 0 --:--:-- --:--:-- --:--:-- 508k dmctl test cmd: "stop-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" got=2 expected=2 dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... got=2 expected=2 [Fri Apr 26 19:15:59 CST 2024] <<<<<< start DM-RemoveLock pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" wait process dm-master.test exit... dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later check sync diff after set SafeModeInitPhaseSeconds failpoint check diff successfully wait for one of DM-worker offine failed, retry later wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for one of DM-worker offine failed, retry later wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time DM-worker1 is sharding lock owner and detects it offline [Fri Apr 26 19:16:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 6.50 secs (484810748 bytes/sec) [Pipeline] { [Pipeline] withEnv [Pipeline] { wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh wait process dm-worker.test exit... wait process dm-master.test exit... [Fri Apr 26 19:16:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... check log contain failed 1-th time, retry later [Pipeline] sh 0 in Sync unit + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:15 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:16 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:15 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:15 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:15 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:15 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:15 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:15 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:15 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:15 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:15 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:15 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:15 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:16 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:15 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:15 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:15 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:15 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:15 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:15 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:15 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:15 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:15 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:15 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh rpc addr 127.0.0.1:8262 is alive wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:15 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:16 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:15 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:15 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:15 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:15 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:15 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:15 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:15 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:15 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:15 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:15 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:15 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:16 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:16 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:15 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:15 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:15 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:16 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:16 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:16 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:16 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:16 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:16 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:16 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:16 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:16 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:16 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:16 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:05 CST 2024] <<<<<< test case incremental_mode success! >>>>>> start running case: [initial_unit] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh... Verbose mode = false dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" check diff successfully dmctl test cmd: "stop-task test" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:05 CST 2024] <<<<<< finish DM-RemoveLock pessimistic >>>>>> [Fri Apr 26 19:16:05 CST 2024] <<<<<< start DM-RemoveLock optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:05 CST 2024] <<<<<< test case ha_cases3 success! >>>>>> start running case: [ha_cases3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:05 CST 2024] <<<<<< start test_isolate_master_and_worker >>>>>> [Fri Apr 26 19:16:05 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" failpoint=github.com/pingcap/tiflow/dm/syncer/LoadCheckpointFailed=return(1152) [Fri Apr 26 19:16:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "config import -p /tmp/configs" wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" dmctl test cmd: "operate-source show" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" import prepare data start DM worker and master [Fri Apr 26 19:16:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:16:07 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:16:07 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 check diff successfully >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_operations passed 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 1-th time, retry later check diff successfully dmctl test cmd: "validation status test" check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/0/source1.yaml" start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Fri Apr 26 19:16:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source1.yaml" [Fri Apr 26 19:16:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Fri Apr 26 19:16:11 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Fri Apr 26 19:16:11 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... wait process dm-worker.test exit... [Fri Apr 26 19:16:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" resume task will also initial failed dmctl test cmd: "resume-task test" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> dmctl test cmd: "validation status test" got=4 expected=4 got=1 expected=1 dmctl test cmd: "validation status --table-stage running test" got=4 expected=4 got=0 expected=0 dmctl test cmd: "validation status --table-stage stopped test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=0 expected=1 command: validation show-error --error all test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:13 CST 2024] <<<<<< test case safe_mode success! >>>>>> start running case: [sequence_safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source2.yaml" start incremental_data wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:16:15 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Fri Apr 26 19:16:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" rpc addr 127.0.0.1:8264 is alive [Fri Apr 26 19:16:16 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status --table-stage stopped" dmctl test cmd: "validation status --table-stage start test" dmctl test cmd: "validation show-error --error resolved test" dmctl test cmd: "validation show-error --error all" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "validation ignore-error test 100 --all" dmctl test cmd: "validation resolve-error test 100 --all" dmctl test cmd: "validation clear-error test 100 --all" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "validation ignore-error test 100 101" dmctl test cmd: "validation resolve-error test 100 101" dmctl test cmd: "validation clear-error test 100 101" dmctl test cmd: "validation ignore-error test error-id" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" dmctl test cmd: "validation resolve-error test error-id" dmctl test cmd: "validation clear-error test error-id" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:16:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source1.yaml" dmctl test cmd: "validation ignore-error test" dmctl test cmd: "validation resolve-error test" rpc addr 127.0.0.1:18262 is alive [Fri Apr 26 19:16:17 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source2.yaml" dmctl test cmd: "validation clear-error test" dmctl test cmd: "validation ignore-error" dmctl test cmd: "validation resolve-error" dmctl test cmd: "validation clear-error" got=2 expected=2 kill dm-worker1 dmctl test cmd: "validation ignore-error non-exist-task-name 1" dmctl test cmd: "validation resolve-error non-exist-task-name 1" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task2.yaml " dmctl test cmd: "validation clear-error non-exist-task-name 1" dmctl test cmd: "validation resolve-error test 1" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation show-error --error unprocessed test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-worker1 exit... start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test 2" dmctl test cmd: "validation status test" [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] stage [Pipeline] { (Test) got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" got=1 expected=1 dmctl test cmd: "validation clear-error test 1" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" dmctl test cmd: "query-status test" got=4 expected=4 got=2 expected=2 dmctl test cmd: "query-status test2" wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:16:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "validation clear-error test 2" dmctl test cmd: "validation status test" got=4 expected=4 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully got=1 expected=1 dmctl test cmd: "validation stop test" got=1 expected=1 dmctl test cmd: "validation status test" check diff successfully wait for port 8262 offline the 1-th time check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=3 expected=3 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 60730 0 60730 0 0 17.7M 0 --:--:-- --:--:-- --:--:-- 19.3M % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 69177 0 69177 0 0 29.3M 0 --:--:-- --:--:-- --:--:-- 32.9M dmctl test cmd: "validation show-error --error all test" check data got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test 3" dmctl test cmd: "validation resolve-error test 4" dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" got=1 expected=1 dmctl test cmd: "validation clear-error test --all" dmctl test cmd: "validation status test" got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_upstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive start dm-worker2 and kill dm-worker1 wait for port 8262 offline the 2-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "validation status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "stop-task test" clean up data dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/dm-task.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker1 exit... [Fri Apr 26 19:16:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:16:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { wait process dm-master.test exit... process dm-master.test already exit [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] sh dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive wait process dm-worker1 exit... process dm-worker1 already exit mysql_ssl_setup at=/var/lib/mysql/ [Fri Apr 26 19:16:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/tls/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] sh got=2 expected=2 --> validation resolve-error --all 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive + '[' TLS_GROUP == G08 ']' + echo 'run G08 test' run G08 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh + make dm_integration_test_in_group GROUP=G08 + '[' TLS_GROUP == G11 ']' + echo 'run G11 test' run G11 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + make dm_integration_test_in_group GROUP=G11 wait process dm-worker.test exit... + '[' TLS_GROUP == G02 ']' + echo 'run G02 test' run G02 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ wait process dm-master.test exit... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + make dm_integration_test_in_group GROUP=G02 + '[' TLS_GROUP == G09 ']' + echo 'run G09 test' run G09 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" resume task to worker2 dmctl test cmd: "resume-task test" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + make dm_integration_test_in_group GROUP=G09 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G08 Run cases: shardddl3 shardddl3_1 shardddl4 shardddl4_1 sharding sequence_sharding ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker.test exit... dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G11 Run cases: adjust_gtid async_checkpoint_flush binlog_parse case_sensitive checkpoint_transaction check_task dm_syncer downstream_diff_index downstream_more_column drop_column_with_index duplicate_event expression_filter extend_column fake_rotate_event foreign_key full_mode gbk gtid ha_cases http_proxies lightning_load_task lightning_mode metrics new_collation_off only_dml openapi s3_dumpling_lightning sequence_sharding_optimistic sequence_sharding_removemeta shardddl_optimistic slow_relay_writer sql_mode sync_collation tracker_ignored_ddl validator_basic validator_checkpoint incompatible_ddl_changes ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) use sync_diff_inspector to check increment data check diff successfully check diff successfully ============================== test_worker_ha_when_enable_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G02 Run cases: handle_error handle_error_2 handle_error_3 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { check diff successfully [Fri Apr 26 19:16:25 CST 2024] <<<<<< finish test_multi_task_running >>>>>> [Fri Apr 26 19:16:25 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time check sync diff after reset failpoint check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G09 Run cases: import_v10x sharding2 ha ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:26 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] container [Pipeline] { dmctl test cmd: "pause-task test" [Pipeline] sh wait process dm-worker.test exit... wait process dm-worker.test exit... + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K Apr 26 11:16 . drwxr-sr-x. 17 1000 1000 4.0K Apr 26 11:16 .. -rw-------. 1 root 1000 1.7K Apr 26 11:16 ca-key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:16 ca.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:16 client-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:16 client-key.pem -rw-------. 1 root 1000 1.7K Apr 26 11:16 private_key.pem -rw-r--r--. 1 root 1000 451 Apr 26 11:16 public_key.pem -rw-r--r--. 1 root 1000 1.1K Apr 26 11:16 server-cert.pem -rw-------. 1 root 1000 1.7K Apr 26 11:16 server-key.pem wait process dm-master.test exit... [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow rpc addr 127.0.0.1:8561 is alive [Pipeline] { [Pipeline] cache version() 5.7.26-log version() 8.0.21 start running case: [shardddl3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [adjust_gtid] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit restart DM-worker after set SequenceShardSyncedExecutionExit failpoint [Fri Apr 26 19:16:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:16:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [import_v10x] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit source1 master status, dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin.000001, 1049, 05750270-03be-11ef-aa8f-0eb410d841b8:1-4 source2 master status, dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin.000001, 1158, 05e06524-03be-11ef-8698-0eb410d841b8:1-4 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source1.yaml" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [handle_error] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:16:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-01 worker1" start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" check diff successfully dmctl test cmd: "resume-task test" dmctl test cmd: "resume-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" 1 dm-master alive wait process dm-master.test exit... got=2 expected=2 finish incremental_data check diff 1 rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source1.yaml" wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 [Fri Apr 26 19:16:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source1.yaml" [Fri Apr 26 19:16:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source1.yaml" wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:16:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:16:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker2.toml >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source2.yaml" wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source2.yaml" wait for one of DM-worker offine failed, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "config task test --path /tmp/dm_test/import_v10x/task.yaml" wait for one of DM-worker offine failed, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" got=1 expected=1 wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-worker.test exit... [Fri Apr 26 19:16:31 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml --remove-meta" DM-worker1 is sharding lock owner and detects it offline [Fri Apr 26 19:16:32 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR >>>>>> [Fri Apr 26 19:16:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status -s mysql-replica-02" [Fri Apr 26 19:16:32 CST 2024] <<<<<< start DM-071 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... got=2 expected=2 got=1 expected=1 [Fri Apr 26 19:16:33 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive wait process dm-master.test exit... dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3" dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb2 add column c int default 100; alter table handle_error.tb2 add primary key (c);" dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "binlog inject test -s mysql-replica-01 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique(c);" dmctl test cmd: "binlog inject test -s mysql-replica-02 alter table handle_error.tb2 drop primary key; alter table handle_error.tb2 add unique(c);" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" run tidb sql failed 1-th time, retry later failpoint=github.com/pingcap/tiflow/dm/syncer/dbconn/GetMasterStatusFailed=return(1152) [Fri Apr 26 19:16:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:16:34 CST 2024] <<<<<< finish DM-071 pessimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status -s mysql-replica-02" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -n worker3" wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:16:35 CST 2024] <<<<<< start DM-073 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "validation status test" wait process dm-master.test exit... process dm-master.test already exit got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/1/source1.yaml" check diff failed 1-th time, retry later wait process dm-worker.test exit... start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" wait process dm-worker.test exit... [Fri Apr 26 19:16:37 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR >>>>>> [Fri Apr 26 19:16:37 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:16:37 CST 2024] <<<<<< finish DM-073 pessimistic >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' [Fri Apr 26 19:16:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master-no-tls.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" wait process dm-master.test exit... check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Fri Apr 26 19:16:38 CST 2024] <<<<<< start DM-076 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:39 CST 2024] <<<<<< test case import_v10x success! >>>>>> start running case: [sharding2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time try to isolate leader dm-master3 kill dm-master3 wait process dm-master3 exit... wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:16:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm) 3149573632 bytes in 9.79 secs (321809775 bytes/sec) [Pipeline] { wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:40 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /tmp/dm_test/tls/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" wait process dm-master3 exit... process dm-master3 already exit [Fri Apr 26 19:16:40 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" resume task will also initial failed dmctl test cmd: "resume-task test" try to isolate follower dm-master2 kill dm-master2 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 [Fri Apr 26 19:16:40 CST 2024] <<<<<< finish DM-076 pessimistic >>>>>> [Fri Apr 26 19:16:40 CST 2024] <<<<<< start DM-076 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-no-tls.yaml" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master2 exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later wait process dm-master2 exit... got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-no-tls.yaml --remove-meta=true" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source1.yaml" [Fri Apr 26 19:16:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> dmctl test cmd: "validation show-error test" got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:16:42 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success check diff successfully [2024/04/26 19:15:25.650 +08:00] [INFO] [dumpling.go:111] ["create dumpling"] [task=test] [unit=dump] [config="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"auto\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"\",\"ServerInfo\":{\"ServerType\":0,\"ServerVersion\":null,\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/04/26 19:15:25.807 +08:00] [INFO] [dump.go:151] ["begin to run Dump"] [task=test] [unit=dump] [conf="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"flush\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"sql\",\"ServerInfo\":{\"ServerType\":1,\"ServerVersion\":\"5.7.26-log\",\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/04/26 19:15:35.671 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14189] [2024/04/26 19:15:45.672 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14189] [2024/04/26 19:15:56.655 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14189] [2024/04/26 19:16:06.655 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=9415377] [2024/04/26 19:16:16.656 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=9904420] [2024/04/26 19:16:26.656 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=15868099] [2024/04/26 19:16:36.657 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=20038264] check diff 2 [Fri Apr 26 19:16:41 CST 2024] <<<<<< finish DM-RemoveLock optimistic >>>>>> wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Fri Apr 26 19:16:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source2.yaml" dmctl test cmd: "query-status test" [Fri Apr 26 19:16:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master2 exit... got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check data check diff successfully ============================== test_source_and_target_with_empty_tlsconfig success ================================== wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:43 CST 2024] <<<<<< test case sequence_safe_mode success! >>>>>> wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> rpc addr 127.0.0.1:8262 is alive [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation resolve-error test --all" dmctl test cmd: "validation show-error test" rpc addr 127.0.0.1:8264 is alive [Fri Apr 26 19:16:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master2 exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:16:44 CST 2024] <<<<<< finish DM-076 optimistic >>>>>> got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 --> validation ignore-error --all 1 dm-master alive [Pipeline] } wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:16:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] sh dmctl test cmd: "start-relay -s mysql-replica-02 worker2" 2 dm-worker alive 0 dm-syncer alive wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] // cache [Pipeline] } rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status -s mysql-replica-01" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:16 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 26 19:16 .. -rw-r--r--. 1 jenkins jenkins 2.0K Apr 26 18:59 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 Apr 26 19:16 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 Apr 26 18:59 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:16 .git drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 .github -rw-r--r--. 1 jenkins jenkins 985 Apr 26 18:59 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 Apr 26 18:59 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K Apr 26 18:59 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K Apr 26 18:59 LICENSE -rw-r--r--. 1 jenkins jenkins 25K Apr 26 18:59 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K Apr 26 18:59 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 Apr 26 18:59 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 README.md -rw-r--r--. 1 jenkins jenkins 2.1K Apr 26 18:59 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K Apr 26 18:59 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K Apr 26 18:59 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K Apr 26 18:59 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:16 bin drwxr-sr-x. 17 jenkins jenkins 4.0K Apr 26 19:16 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:16 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K Apr 26 19:16 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K Apr 26 19:16 dm drwxr-sr-x. 7 jenkins jenkins 4.0K Apr 26 19:16 docs drwxr-sr-x. 13 jenkins jenkins 4.0K Apr 26 19:16 engine -rwxr-xr-x. 1 jenkins jenkins 26K Apr 26 18:59 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 examples -rw-r--r--. 1 jenkins jenkins 20K Apr 26 18:59 go.mod -rw-r--r--. 1 jenkins jenkins 151K Apr 26 18:59 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K Apr 26 19:16 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:16 proto drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 26 19:16 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K Apr 26 19:16 tests -rw-r--r--. 1 jenkins jenkins 582 Apr 26 18:59 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 26 19:16 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process dm-master2 exit... [Pipeline] // dir [Fri Apr 26 19:16:45 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=3 command: query-status -s mysql-replica-02 "result": true count: 2 != expected: 3, failed the 0-th time, will retry again [Pipeline] } [Pipeline] } got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " wait process dm-master.test exit... [Pipeline] // timeout [Pipeline] // withCredentials [Pipeline] } [Pipeline] sh rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" [Pipeline] // timeout wait process dm-master2 exit... [Pipeline] } + '[' TLS_GROUP == G00 ']' + echo 'run G00 test' run G00 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] // stage [Pipeline] } got=2 expected=2 [Fri Apr 26 19:16:46 CST 2024] <<<<<< start DM-RestartMaster pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait for port 8262 offline the 1-th time [Pipeline] // container [Pipeline] } + make dm_integration_test_in_group GROUP=G00 wait process dm-master.test exit... process dm-master.test already exit [Pipeline] // withEnv VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Pipeline] } dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] // node [Pipeline] } check sync diff for the increment replication check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " got=1 expected=1 dmctl test cmd: "stop-task test" [Pipeline] // podTemplate wait process dm-master2 exit... process dm-master2 already exit [Fri Apr 26 19:16:47 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> [Pipeline] } [Pipeline] // withEnv [Fri Apr 26 19:16:47 CST 2024] <<<<<< finish DM-077 >>>>>> [Fri Apr 26 19:16:47 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" [Pipeline] } "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) try to isolate dm-worker1 kill dm-worker1 [Pipeline] // stage [Pipeline] } wait process dm-worker.test exit... wait process dm-master.test exit... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G00 Run cases: ha_cases_1 ha_cases_2 ha_cases2 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:16:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member --worker" >>>>>>>>>>>>>>>>>>>>>>>>>>test test_restart_relay_status passed 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive check diff successfully check log contain failed 1-th time, retry later check diff successfully wait process dm-worker1 exit... check sync diff for the increment replication wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:16:49 CST 2024] <<<<<< finish DM-077 >>>>>> wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:16:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> try to isolate dm-worker2 kill dm-worker2 rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:50 CST 2024] <<<<<< start test_kill_master >>>>>> [Fri Apr 26 19:16:50 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:16:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully kill dm-worker1 [Fri Apr 26 19:16:50 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker2 exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:16:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> dmctl test cmd: "pause-task test" got=3 expected=3 dmctl test cmd: "resume-task test" wait process dm-master exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:52 CST 2024] <<<<<< test case tls success! >>>>>> got=1 expected=1 check diff failed 1-th time, retry later got=3 expected=3 wait process dm-master.test exit... process dm-master.test already exit [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) wait process dm-worker1 exit... [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // withCredentials import prepare data start DM worker and master cluster [Fri Apr 26 19:16:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:16:52 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:16:52 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-master exit... process dm-master already exit [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] // stage [Pipeline] } wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker2 exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:16:54 CST 2024] <<<<<< finish DM-078 >>>>>> [Fri Apr 26 19:16:54 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:16:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:16:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Fri Apr 26 19:16:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:55 CST 2024] <<<<<< test case adjust_gtid success! >>>>>> start running case: [async_checkpoint_flush] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:16:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:16:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:16:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" [Fri Apr 26 19:16:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive wait for port 8262 offline the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/async_checkpoint_flush/source1.yaml" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-task.yaml " got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" check diff successfully dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" kill dm-worker1 [Fri Apr 26 19:16:58 CST 2024] <<<<<< finish DM-RestartMaster pessimistic >>>>>> [Fri Apr 26 19:16:58 CST 2024] <<<<<< start DM-RestartMaster optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:16:58 CST 2024] <<<<<< finish DM-078 >>>>>> wait for port 8262 offline the 2-th time dmctl test cmd: "query-status test" got=1 expected=1 start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " check diff successfully PID of insert_data is 1987 wait process dm-worker1 exit... wait process dm-worker.test exit... [Fri Apr 26 19:16:59 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" [Fri Apr 26 19:16:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully wait process tidb-server exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:17:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process tidb-server exit... process tidb-server already exit wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process tidb-server exit... process tidb-server already exit [Fri Apr 26 19:17:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" [Fri Apr 26 19:17:01 CST 2024] <<<<<< finish DM-079 >>>>>> [Fri Apr 26 19:17:01 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" check diff successfully wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:01 CST 2024] <<<<<< test case initial_unit success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] // cache check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=2 expected=2 got=1 expected=1 restart dm-master [Pipeline] // container got=1 expected=1 got=1 expected=1 Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 [Pipeline] // withEnv [Pipeline] } [Pipeline] // node dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:01 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> [Fri Apr 26 19:17:01 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] } got=1 expected=1 dmctl test cmd: "stop-task test" [Pipeline] // stage [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master exit... dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=2 check sync diff for the increment replication check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 1 dm-master alive 2 dm-worker alive [Fri Apr 26 19:17:03 CST 2024] <<<<<< finish DM-079 >>>>>> got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" 0 dm-syncer alive use sync_diff_inspector to check increment data check diff successfully [Fri Apr 26 19:17:03 CST 2024] <<<<<< finish test_running >>>>>> kill dm-master1 check log contain failed 1-th time, retry later dmctl test cmd: "validation show-error test" got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again [Fri Apr 26 19:17:04 CST 2024] <<<<<< start DM-080 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master exit... process dm-master already exit VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. wait process dm-master.test exit... ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_downstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master1 exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" run tidb sql failed 1-th time, retry later got=4 expected=4 dmctl test cmd: "stop-task test" wait process dm-master1 exit... process dm-master1 already exit waiting 5 seconds [Fri Apr 26 19:17:06 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> [Fri Apr 26 19:17:06 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test --all" dmctl test cmd: "validation show-error test" got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 wait process dm-master.test exit... process dm-master.test already exit 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Fri Apr 26 19:17:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01,mysql-replica-02 alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=4 expected=4 run tidb sql failed 1-th time, retry later [Fri Apr 26 19:17:08 CST 2024] <<<<<< finish DM-080 pessimistic >>>>>> wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:08 CST 2024] <<<<<< test case sharding2 success! >>>>>> start running case: [ha] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM worker and master [Fri Apr 26 19:17:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:17:08 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" [Fri Apr 26 19:17:09 CST 2024] <<<<<< start DM-081 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:17:09 CST 2024] <<<<<< finish DM-RestartMaster optimistic >>>>>> [Fri Apr 26 19:17:09 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:10 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> [Fri Apr 26 19:17:10 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:17:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later check task is running HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive check master2,3 are running dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:17:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8361 is alive wait for valid metric of start_leader_counter for 1-th time, got value: '0' check diff successfully got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:13 CST 2024] <<<<<< finish DM-081 pessimistic >>>>>> got=0 expected=1 command: query-status -s mysql-replica-01 "relayCatchUpMaster": true count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:17:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for valid metric of start_leader_counter for 2-th time, got value: '0' use sync_diff_inspector to check increment2 data now! check diff successfully [Fri Apr 26 19:17:13 CST 2024] <<<<<< finish test_kill_master >>>>>> [Fri Apr 26 19:17:13 CST 2024] <<<<<< start test_kill_and_isolate_worker >>>>>> [Fri Apr 26 19:17:13 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... [Fri Apr 26 19:17:14 CST 2024] <<<<<< start DM-082 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source1.yaml" got=4 expected=4 dmctl test cmd: "stop-task test" wait for valid metric of start_leader_counter for 3-th time, got value: '0' [Fri Apr 26 19:17:14 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> [Fri Apr 26 19:17:14 CST 2024] <<<<<< start DM-LIST_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check diff failed 1-th time, retry later metric start_leader_counter has invalid value 0 [Fri Apr 26 19:17:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_kill_dump_connection passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "binlog list test" rpc addr 127.0.0.1:8262 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha/source1.yaml" dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:14435" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog inject test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:14627 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique (c);" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog list test" wait process dm-master.test exit... dmctl test cmd: "binlog skip test" run tidb sql failed 1-th time, retry later check diff successfully wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later [Fri Apr 26 19:17:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 --> (fail) validation start: missing mode value dmctl test cmd: "validation start --mode" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:18 CST 2024] <<<<<< finish DM-082 pessimistic >>>>>> wait process dm-master.test exit... --> (fail) validation start: invalid mode dmctl test cmd: "validation start --mode xxx" --> (fail) validation start: missing start-time value dmctl test cmd: "validation start --start-time" wait process dm-worker.test exit... --> (fail) validation start: invalid start-time dmctl test cmd: "validation start --start-time xx" --> (fail) validation start: without all-task and task-name dmctl test cmd: "validation start --mode full" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha/source2.yaml" --> (fail) validation start: with both all-task and task-name dmctl test cmd: "validation start --all-task test" --> (fail) validation start: too many arguments dmctl test cmd: "validation start test test2" --> (fail) validation start: non-existed subtask dmctl test cmd: "validation start xxxxx" [Fri Apr 26 19:17:19 CST 2024] <<<<<< start DM-085 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" wait process dm-master.test exit... --> (fail) validation start: non-exist source dmctl test cmd: "validation start -s xxxxx --all-task" --> (fail) validation stop: without all-task and task-name dmctl test cmd: "validation stop" --> (fail) validation stop: with both all-task and task-name dmctl test cmd: "validation stop --all-task test" --> (fail) validation stop: too many arguments dmctl test cmd: "validation stop test test2" wait process dm-worker.test exit... --> (fail) validation stop: non-existed subtask dmctl test cmd: "validation stop xxxxx" --> (fail) validation stop: non-exist source dmctl test cmd: "validation stop -s xxxxx --all-task" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:19 CST 2024] <<<<<< finish DM-LIST_ERROR >>>>>> [Fri Apr 26 19:17:19 CST 2024] <<<<<< start DM-SKIP_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" --> (fail) validation stop: stop not-enabled validator dmctl test cmd: "validation stop -s mysql-replica-01 test" --> (success) validation start: start validation without explicit mode for source 1 dmctl test cmd: "validation start -s mysql-replica-01 test" wait process dm-master.test exit... --> (fail) validation start: start all validator with explicit mode, but 1 subtask already enabled dmctl test cmd: "validation start --mode full test" --> (fail) validation start: start validation with explicit mode for source 1 again dmctl test cmd: "validation start -s mysql-replica-01 --mode full test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:20 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS pessimistic >>>>>> [Fri Apr 26 19:17:20 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:20 CST 2024] <<<<<< test case new_relay success! >>>>>> start running case: [all_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:20 CST 2024] <<<<<< start test_expression_filter >>>>>> --> (fail) validation start: start all validator without explicit mode dmctl test cmd: "validation start test" --> (fail) validation stop: stop all but 1 subtask is not enabled dmctl test cmd: "validation stop test" dmctl test cmd: "query-status test" got=1 expected=1 [Fri Apr 26 19:17:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time --> (success) validation start: start validation with fast mode and start-time for source 2 dmctl test cmd: "validation start -s mysql-replica-02 --mode fast --start-time '2020-01-01 00:01:00' test" dmctl test cmd: "validation status test" check diff failed 1-th time, retry later got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation start: start all validator of the task without explicit param again, i.e. resuming dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=2 expected=2 [Fri Apr 26 19:17:21 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation stop: stop validation of source 1 dmctl test cmd: "validation stop -s mysql-replica-01 test" dmctl test cmd: "validation status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> (success) validation stop: stop all of test dmctl test cmd: "validation stop test" dmctl test cmd: "validation status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:17:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation stop: stop all of test again dmctl test cmd: "validation stop test" dmctl test cmd: "validation status test" check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: start all of test dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8461 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml " [Fri Apr 26 19:17:22 CST 2024] <<<<<< finish DM-SKIP_ERROR >>>>>> [Fri Apr 26 19:17:22 CST 2024] <<<<<< start DM-REPLACE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation stop: multiple source dmctl test cmd: "validation stop -s mysql-replica-01 -s mysql-replica-02 test" dmctl test cmd: "validation status test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: multiple source dmctl test cmd: "validation start -s mysql-replica-01 -s mysql-replica-02 test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:17:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:23 CST 2024] <<<<<< finish DM-085 pessimistic >>>>>> [Fri Apr 26 19:17:23 CST 2024] <<<<<< start DM-085 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase dmctl test cmd: "query-status test" got=2 expected=2 use sync_diff_inspector to check increment data check diff failed 1-th time, retry later check diff successfully rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column new_col text, add column c int; alter table handle_error.tb1 add unique(c);" check diff failed 1-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2-no-validator.yaml --remove-meta" import prepare data start DM worker and master cluster [Fri Apr 26 19:17:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:17:24 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:17:24 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" [Fri Apr 26 19:17:25 CST 2024] <<<<<< finish DM-085 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Fri Apr 26 19:17:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 --> (fail) validation update: on non-existed validator dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid d5226f57-03bd-11ef-a49f-6a11d69e995d:1-999" --> (success) validation start: start all tasks dmctl test cmd: "validation start --all-task --mode fast" --> (fail) validation update: missing task-name dmctl test cmd: "validation update" --> (fail) validation update: multi tasks dmctl test cmd: "validation update test test2" dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:26 CST 2024] <<<<<< start DM-086 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" start dm-worker3 and kill dm-worker2 --> (fail) validation update: non-existed subtask dmctl test cmd: "validation update xxxxx" --> (fail) validation update: non-exist source dmctl test cmd: "validation update -s xxxxx test" [Fri Apr 26 19:17:26 CST 2024] <<<<<< finish DM-REPLACE_ERROR >>>>>> [Fri Apr 26 19:17:26 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully check diff successfully check diff successfully dmctl test cmd: "stop-task test" --> (fail) validation update: not specify cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-02" --> (fail) validation update: not specify cutover-binlog-pos dmctl test cmd: "validation update test2 -s mysql-replica-01" --> (fail) validation update: invalid cutover-binlog-pos dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos xxx" --> (fail) validation update: specify both cutover-binlog-pos and cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos '(mysql-bin.000001, 2345)' --cutover-binlog-gtid d5226f57-03bd-11ef-a49f-6a11d69e995d:1-999" check diff successfully check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" --> (success) validation update: on exist source dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid d5226f57-03bd-11ef-a49f-6a11d69e995d:1-999" dmctl test cmd: "validation status test" [Fri Apr 26 19:17:27 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS optimistic >>>>>> [Fri Apr 26 19:17:27 CST 2024] <<<<<< start DM-COLUMN_INDEX pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker2 exit... got=1 expected=1 got=0 expected=0 got=2 expected=2 got=2 expected=2 got=0 expected=0 got=2 expected=2 got=1 expected=1 dmctl test cmd: "validation status test2" got=1 expected=1 got=0 expected=0 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 --> (success) validation stop: stop all tasks dmctl test cmd: "validation stop --all-task" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=0 got=2 expected=2 got=0 expected=0 got=2 expected=2 dmctl test cmd: "validation status test2" got=1 expected=1 dmctl test cmd: "query-status test" [Fri Apr 26 19:17:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:17:28 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time got=4 expected=4 run tidb sql failed 1-th time, retry later [Fri Apr 26 19:17:28 CST 2024] <<<<<< finish DM-086 pessimistic >>>>>> [Fri Apr 26 19:17:28 CST 2024] <<<<<< start DM-086 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 2-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker2" check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:30 CST 2024] <<<<<< finish DM-086 optimistic >>>>>> start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " [Fri Apr 26 19:17:30 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING pessimistic >>>>>> [Fri Apr 26 19:17:30 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh: line 29: 1987 Terminated insert_data 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... check diff failed 3-th time, retry later [Fri Apr 26 19:17:31 CST 2024] <<<<<< start DM-094 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" [Fri Apr 26 19:17:31 CST 2024] <<<<<< finish DM-COLUMN_INDEX pessimistic >>>>>> [Fri Apr 26 19:17:31 CST 2024] <<<<<< start DM-COLUMN_INDEX optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=4 expected=4 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:17:32 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING optimistic >>>>>> [Fri Apr 26 19:17:32 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:33 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed at last dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:17:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:34 CST 2024] <<<<<< test case async_checkpoint_flush success! >>>>>> start running case: [binlog_parse] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=4 expected=4 run tidb sql failed 1-th time, retry later got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:34 CST 2024] <<<<<< finish DM-094 pessimistic >>>>>> [Fri Apr 26 19:17:34 CST 2024] <<<<<< start DM-094 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:17:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time use sync_diff_inspector to check increment data check diff successfully [Fri Apr 26 19:17:35 CST 2024] <<<<<< finish test_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:17:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/binlog_parse/source1.yaml" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml" [Fri Apr 26 19:17:36 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> [Fri Apr 26 19:17:36 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 kill dm-worker2 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait process dm-worker.test exit... prepare data start task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-task.yaml --remove-meta" wait process dm-worker2 exit... wait for the task to be scheduled and keep paused HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive resume task before kill and restart dm-worker dmctl test cmd: "resume-task test" start dm-worker2 and kill dm-worker3 got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:38 CST 2024] <<<<<< finish DM-COLUMN_INDEX optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" [Fri Apr 26 19:17:38 CST 2024] <<<<<< finish DM-094 optimistic >>>>>> got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" dmctl test cmd: "check-task /tmp/dm_test/dmctl_command/temp.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:38 CST 2024] <<<<<< finish test_expression_filter >>>>>> [Fri Apr 26 19:17:38 CST 2024] <<<<<< start test_json_expression >>>>>> [Fri Apr 26 19:17:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later wait process dm-worker3 exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 [Fri Apr 26 19:17:38 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time check full phase check diff successfully dmctl test cmd: "validation start test" prepare incremental data check incremental phase % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 122k 0 --:--:-- --:--:-- --:--:-- 104k check diff successfully dmctl test cmd: "start-relay -s mysql-replica-01 worker1" [Fri Apr 26 19:17:39 CST 2024] <<<<<< start DM-095 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:17:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker exit... wait process dm-worker3 exit... process dm-worker3 already exit [Fri Apr 26 19:17:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8264 is alive wait and check task running wait for rpc addr 127.0.0.1:8263 alive the 1-th time HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Fri Apr 26 19:17:40 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time got=2 expected=2 got=1 expected=1 got=1 expected=1 prepare incremental data 2 dmctl test cmd: "validation start test" wait process dm-worker exit... process dm-worker already exit [Fri Apr 26 19:17:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:17:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check incremental phase 2 check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:17:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:17:40 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> [Fri Apr 26 19:17:40 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "pause-task test" rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3 worker4" dmctl test cmd: "resume-task test" got=3 expected=3 rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:17:41 CST 2024] <<<<<< start DM-DML_EXECUTE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 restart dm-worker3 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 87872 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" wait process dm-worker3 exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:43 CST 2024] <<<<<< test case dmctl_command success! >>>>>> [Pipeline] } [Fri Apr 26 19:17:43 CST 2024] <<<<<< finish DM-095 pessimistic >>>>>> [Fri Apr 26 19:17:43 CST 2024] <<<<<< start DM-095 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker3 exit... Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) process dm-worker3 already exit wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Fri Apr 26 19:17:44 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time [Pipeline] // cache wait process dm-worker.test exit... [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:45 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP pessimistic >>>>>> [Fri Apr 26 19:17:45 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // withEnv dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully [Pipeline] } dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:45 CST 2024] <<<<<< finish DM-DML_EXECUTE_ERROR >>>>>> [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" [Pipeline] // withEnv [Pipeline] } wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:17:45 CST 2024] <<<<<< test case binlog_parse success! >>>>>> start running case: [case_sensitive] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Pipeline] // stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] } rpc addr 127.0.0.1:8264 is alive isolate dm-worker4 kill dm-worker4 wait process dm-worker exit... [Fri Apr 26 19:17:46 CST 2024] <<<<<< finish DM-095 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8261 is alive wait process dm-worker4 exit... [Fri Apr 26 19:17:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Fri Apr 26 19:17:47 CST 2024] <<<<<< start DM-096 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker exit... process dm-worker already exit [Fri Apr 26 19:17:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:17:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker4 exit... process dm-worker4 already exit [Fri Apr 26 19:17:47 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 isolate dm-worker3 kill dm-worker3 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" [Fri Apr 26 19:17:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:17:48 CST 2024] <<<<<< start DM-KEY_NOT_FOUND >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Fri Apr 26 19:17:48 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 117 100 117 0 0 205k 0 --:--:-- --:--:-- --:--:-- 114k {"connections":0,"version":"8.0.11-TiDB-v7.5.1-43-ge406d5780b","git_hash":"e406d5780b18a1f2aaf6230cde4b6403991e228d"}dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task-2.yaml " wait process dm-worker3 exit... got=3 expected=3 check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:49 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP optimistic >>>>>> [Fri Apr 26 19:17:49 CST 2024] <<<<<< start DM-SKIP_INCOMPATIBLE_DDL >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker3" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "list-member --name worker3" got=1 expected=1 query-status from all dm-master dmctl test cmd: "query-status test" wait process dm-worker3 exit... process dm-worker3 already exit [Fri Apr 26 19:17:49 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 disable isolate dm-worker4 kill dm-worker4 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "query-status test" [Fri Apr 26 19:17:50 CST 2024] <<<<<< finish DM-KEY_NOT_FOUND >>>>>> run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh: line 556: hello: command not found dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 join new dm-master and query-status [Fri Apr 26 19:17:50 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master4.toml >>>>>> got=2 expected=2 got=1 expected=1 dmctl test cmd: "binlog skip test" wait for rpc addr 127.0.0.1:8561 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-worker exit... [Fri Apr 26 19:17:51 CST 2024] <<<<<< finish DM-SKIP_INCOMPATIBLE_DDL >>>>>> [Fri Apr 26 19:17:51 CST 2024] <<<<<< start DM-4202 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker4 exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:51 CST 2024] <<<<<< finish DM-096 pessimistic >>>>>> [Fri Apr 26 19:17:51 CST 2024] <<<<<< start DM-096 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "query-status test" check diff successfully got=3 expected=3 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... [Fri Apr 26 19:17:52 CST 2024] <<<<<< finish test_json_expression >>>>>> [Fri Apr 26 19:17:52 CST 2024] <<<<<< start test_fail_job_between_event >>>>>> [Fri Apr 26 19:17:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker4 exit... process dm-worker4 already exit [Fri Apr 26 19:17:52 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=3 command: query-status test "stage": "Running" count: 1 != expected: 3, failed the 0-th time, will retry again wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait process dm-worker exit... process dm-worker already exit [Fri Apr 26 19:17:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:17:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:35724" run tidb sql failed 2-th time, retry later wait pattern dm-worker1.toml exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Fri Apr 26 19:17:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:17:53 CST 2024] <<<<<< finish DM-4202 >>>>>> rpc addr 127.0.0.1:8261 is alive check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:17:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:17:53 CST 2024] <<<<<< start DM-001 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" [Fri Apr 26 19:17:53 CST 2024] <<<<<< finish DM-096 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Fri Apr 26 19:17:54 CST 2024] <<<<<< start DM-4204 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait pattern dm-worker2.toml exit... run tidb sql failed 3-th time, retry later wait pattern dm-worker2.toml exit... got=3 expected=3 query-status from all dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:54 CST 2024] <<<<<< start DM-097 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait pattern dm-worker2.toml exit... got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "pause-task test" wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... restart worker4 dmctl test cmd: "query-status test" [Fri Apr 26 19:17:55 CST 2024] <<<<<< finish DM-001 >>>>>> wait pattern dm-worker2.toml exit... got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" [Fri Apr 26 19:17:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Fri Apr 26 19:17:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:56 CST 2024] <<<<<< finish DM-4204 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:17:56 CST 2024] <<<<<< start DM-002 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" run tidb sql failed 4-th time, retry later check diff successfully rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-worker4 exit... rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:17:57 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time wait process dm-master exit... [Fri Apr 26 19:17:57 CST 2024] <<<<<< start DM-4206 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker4 exit... process dm-worker4 already exit [Fri Apr 26 19:17:57 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:17:58 CST 2024] <<<<<< finish DM-002 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 5-th time, retry later wait process dm-master exit... process dm-master already exit dmctl test cmd: "query-status test" rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "resume-task test" [Fri Apr 26 19:17:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:39469 -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:39276 -s mysql-replica-01" [Fri Apr 26 19:17:59 CST 2024] <<<<<< start DM-003 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:36408 -s mysql-replica-02" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:36627 -s mysql-replica-02" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" [Fri Apr 26 19:18:00 CST 2024] <<<<<< finish DM-4206 pessimistic >>>>>> [Fri Apr 26 19:18:00 CST 2024] <<<<<< start DM-4206 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" got=1 expected=1 check diff failed 1-th time, retry later run tidb sql failed 6-th time, retry later rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" use sync_diff_inspector to check increment2 data now! check diff successfully [Fri Apr 26 19:18:00 CST 2024] <<<<<< finish test_kill_and_isolate_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=3 expected=3 check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Fri Apr 26 19:18:01 CST 2024] <<<<<< finish DM-097 pessimistic >>>>>> [Fri Apr 26 19:18:01 CST 2024] <<<<<< start DM-097 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 wait process dm-master.test exit... dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:41438 -s mysql-replica-01" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:41245 -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:38566 -s mysql-replica-02" dmctl test cmd: "query-status test" run tidb sql failed 7-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:38785 -s mysql-replica-02" dmctl test cmd: "query-status test" check diff failed 2-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit check diff successfully [Fri Apr 26 19:18:03 CST 2024] <<<<<< finish DM-4206 optimistic >>>>>> [Fri Apr 26 19:18:03 CST 2024] <<<<<< START DM-MASTER on port 8761, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master6.toml >>>>>> wait for rpc addr 127.0.0.1:8761 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master exit... [Fri Apr 26 19:18:04 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8761 is alive dmctl test cmd: "query-status test" run tidb sql failed 8-th time, retry later got=3 expected=3 check diff failed 3-th time, retry later wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit [Fri Apr 26 19:18:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:43214" dmctl test cmd: "query-status test" Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:40724" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" check diff failed at last check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Fri Apr 26 19:18:07 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Fri Apr 26 19:18:07 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Fri Apr 26 19:18:07 CST 2024] <<<<<< finish DM-003 pessimistic >>>>>> check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:08 CST 2024] <<<<<< finish test_fail_job_between_event >>>>>> [Fri Apr 26 19:18:08 CST 2024] <<<<<< start test_session_config >>>>>> [Fri Apr 26 19:18:08 CST 2024] <<<<<< start DM-004 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... [Fri Apr 26 19:18:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:44990" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:42663" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... got=3 expected=3 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later [Fri Apr 26 19:18:09 CST 2024] <<<<<< finish DM-097 optimistic >>>>>> wait process dm-master.test exit... check diff successfully kill dm-master1 wait process dm-master.test exit... [Fri Apr 26 19:18:10 CST 2024] <<<<<< start DM-098 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:18:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-master1 exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:11 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> [Fri Apr 26 19:18:11 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... wait process dm-master1 exit... wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:12 CST 2024] <<<<<< finish DM-004 optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "query-status test" wait process dm-worker.test exit... wait process dm-master1 exit... process dm-master1 already exit kill dm-master2 got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... [Fri Apr 26 19:18:13 CST 2024] <<<<<< start DM-005 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker1 exit... dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:47024" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:44602" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:13 CST 2024] <<<<<< test case many_tables success! >>>>>> start running case: [online_ddl] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master2 exit... [Fri Apr 26 19:18:14 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Fri Apr 26 19:18:14 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:18:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" check diff successfully dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master2 exit... process dm-master2 already exit [Fri Apr 26 19:18:14 CST 2024] <<<<<< finish DM-005 >>>>>> initial cluster of dm-masters have been killed now we will check whether joined masters can work normally dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive wait process dm-master.test exit... dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" command: query-status test task test has no source or not exist count: 0 != expected: 1 can not query test task's status(in sources []) fail to run: [query-status test] Error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:8361: connect: connection refused" PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:18:15 CST 2024] <<<<<< finish DM-098 pessimistic >>>>>> wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:49058" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source1.yaml" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:46541" dmctl test cmd: "query-status test" wait process dm-master.test exit... dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" [Fri Apr 26 19:18:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> [Fri Apr 26 19:18:16 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" command: query-status test task test has no source or not exist count: 0 != expected: 1 { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-01", "worker": "worker3", "result": null, "relayStatus": null }, "subTaskStatus": [ { "name": "test", "stage": "Running", "unit": "Sync", "result": null, "unresolvedDDLLockID": "", "sync": { "totalEvents": "1", "totalTps": "0", "recentTps": "0", "masterBinlog": "(dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin.000002, 5693)", "masterBinlogGtid": "05750270-03be-11ef-aa8f-0eb410d841b8:1-24", "syncerBinlog": "(dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin.000002, 5693)", "syncerBinlogGtid": "05750270-03be-11ef-aa8f-0eb410d841b8:1-24", "blockingDDLs": [ ], "unresolvedGroups": [ ], "synced": true, "binlogType": "remote", "secondsBehindMaster": "0", "blockDDLOwner": "", "conflictMsg": "", "totalRows": "1", "totalRps": "0", "recentRps": "0" }, "validation": null } ] }, { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-02", "worker": "worker2", "result": null, "relayStatus": { "masterBinlog": "(dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin.000002, 487)", "masterBinlogGtid": "05e06524-03be-11ef-8698-0eb410d841b8:1-5", "relaySubDir": "05e06524-03be-11ef-8698-0eb410d841b8.000001", "relayBinlog": "(dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin.000002, 487)", "relayBinlogGtid": "05e06524-03be-11ef-8698-0eb410d841b8:1-5", "relayCatchUpMaster": true, "stage": "Running", "result": null } }, "subTaskStatus": [ { "name": "test", "stage": "Running", "unit": "Sync", "result": null, "unresolvedDDLLockID": "", "sync": { "totalEvents": "1", "totalTps": "0", "recentTps": "0", "masterBinlog": "(dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin.000002, 487)", "masterBinlogGtid": "05e06524-03be-11ef-8698-0eb410d841b8:1-5", "syncerBinlog": "(dm-it-7976ebdf-ae83-4dff-9dc9-39d79086abe1-5jm5v-djw9h-bin|000001.000002, 487)", "syncerBinlogGtid": "05e06524-03be-11ef-8698-0eb410d841b8:1-5", "blockingDDLs": [ ], "unresolvedGroups": [ ], "synced": true, "binlogType": "local", "secondsBehindMaster": "0", "blockDDLOwner": "", "conflictMsg": "", "totalRows": "1", "totalRps": "0", "recentRps": "0" }, "validation": null } ] } ] } PASS coverage: 3.5% of statements in github.com/pingcap/tiflow/dm/... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:18:17 CST 2024] <<<<<< start DM-4209 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source2.yaml" wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "start-task /tmp/dm_test/online_ddl/dm-task.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:19 CST 2024] <<<<<< finish test_session_config >>>>>> [Fri Apr 26 19:18:19 CST 2024] <<<<<< start test_query_timeout >>>>>> [Fri Apr 26 19:18:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait process dm-worker.test exit... [Fri Apr 26 19:18:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... use sync_diff_inspector to check full dump data check diff successfully alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:20 CST 2024] <<<<<< test case shardddl1 success! >>>>>> start running case: [shardddl1_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml" process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:21 CST 2024] <<<<<< test case shardddl3 success! >>>>>> start running case: [shardddl3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh... Verbose mode = false use sync_diff_inspector to check increment2 data now! check diff successfully 4 dm-master alive 3 dm-worker alive 0 dm-syncer alive dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:21 CST 2024] <<<<<< finish DM-4209 pessimistic >>>>>> [Fri Apr 26 19:18:21 CST 2024] <<<<<< start DM-4209 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Fri Apr 26 19:18:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source1.yaml" check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source1.yaml" [Fri Apr 26 19:18:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source2.yaml" wait process dm-master.test exit... [Fri Apr 26 19:18:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:25 CST 2024] <<<<<< finish DM-4209 optimistic >>>>>> check diff successfully wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source2.yaml" wait pattern dm-worker1.toml exit... [Fri Apr 26 19:18:26 CST 2024] <<<<<< start DM-SAME_DDL_TWICE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait process dm-master.test exit... [Fri Apr 26 19:18:26 CST 2024] <<<<<< start DM-4211 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait pattern dm-worker1.toml exit... wait process dm-master.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Fri Apr 26 19:18:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:18:27 CST 2024] <<<<<< start DM-099 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 use sync_diff_inspector to check increment data check diff successfully [Fri Apr 26 19:18:23 CST 2024] <<<<<< finish test_isolate_master_and_worker >>>>>> 5 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. check diff failed 1-th time, retry later dmctl test cmd: "binlog replace test --binlog-pos dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:54386 alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 wait pattern dm-worker2.toml exit... got=2 expected=2 dmctl test cmd: "stop-task test" wait pattern dm-worker2.toml exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-master.test exit... [Fri Apr 26 19:18:29 CST 2024] <<<<<< finish DM-4211 >>>>>> wait process dm-master.test exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Fri Apr 26 19:18:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:18:30 CST 2024] <<<<<< start DM-4213 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker2 exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:31 CST 2024] <<<<<< finish DM-SAME_DDL_TWICE pessimistic >>>>>> [Fri Apr 26 19:18:31 CST 2024] <<<<<< start DM-006 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:31 CST 2024] <<<<<< test case ha success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 [Pipeline] // cache [Pipeline] } wait process dm-master.test exit... [Pipeline] // dir [Pipeline] } rpc addr 127.0.0.1:8263 is alive wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:18:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" wait process dm-master.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; [Pipeline] // withCredentials [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // timeout pt-online-schema-change 3.0.12 [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } dmctl test cmd: "stop-task test" ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Fri Apr 26 19:18:32 CST 2024] <<<<<< finish DM-4213 >>>>>> wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:32 CST 2024] <<<<<< test case ha_cases3_1 success! >>>>>> start running case: [ha_master] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit start DM worker and master [Fri Apr 26 19:18:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:18:32 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:18:32 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> [Fri Apr 26 19:18:32 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> [Fri Apr 26 19:18:32 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // stage [Pipeline] } check diff failed 1-th time, retry later [Fri Apr 26 19:18:32 CST 2024] <<<<<< finish DM-099 pessimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive rpc addr 127.0.0.1:8561 is alive rpc addr 127.0.0.1:8661 is alive [Fri Apr 26 19:18:33 CST 2024] <<<<<< start DM-100 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 [Fri Apr 26 19:18:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:18:35 CST 2024] <<<<<< finish DM-006 >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-master.test exit... [Fri Apr 26 19:18:36 CST 2024] <<<<<< start DM-007 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive use sync_diff_inspector to check full data check diff successfully alter: add column age int; 1.1.0 wait process dm-worker.test exit... wait process dm-worker1 exit... wait process dm-worker.test exit... wait process dm-worker1 exit... process dm-worker1 already exit wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Fri Apr 26 19:18:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:38 CST 2024] <<<<<< test case handle_error success! >>>>>> start running case: [handle_error_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" alter: add column age int; 1.1.0 [Fri Apr 26 19:18:39 CST 2024] <<<<<< finish DM-100 optimistic >>>>>> kill dm-master1 and kill dm-master2 wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Fri Apr 26 19:18:40 CST 2024] <<<<<< start DM-101 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master1 exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:39 CST 2024] <<<<<< finish DM-007 >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time alter: add key name (name); 1.1.0 wait process dm-master1 exit... process dm-master1 already exit [Fri Apr 26 19:18:40 CST 2024] <<<<<< start DM-008 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source1.yaml" check diff successfully [Fri Apr 26 19:18:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master2 exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker2 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source2.yaml" wait process dm-master2 exit... process dm-master2 already exit master4 [Fri Apr 26 19:18:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time alter: add key name (name); 1.1.0 wait process dm-master.test exit... check diff successfully wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:18:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:18:44 CST 2024] <<<<<< start DM-4215 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 2-th time rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:44 CST 2024] <<<<<< finish DM-008 >>>>>> alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 [Fri Apr 26 19:18:45 CST 2024] <<<<<< finish DM-101 optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8262 alive the 3-th time dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" [Fri Apr 26 19:18:45 CST 2024] <<<<<< start DM-009 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8262 alive the 4-th time [Fri Apr 26 19:18:46 CST 2024] <<<<<< start DM-102 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8262 alive the 5-th time got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8262 alive the 6-th time wait process dm-master.test exit... dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb`" wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8262 alive the 7-th time wait process dm-master.test exit... alter: add column age int; 1.1.0 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:49 CST 2024] <<<<<< finish DM-009 >>>>>> wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Fri Apr 26 19:18:50 CST 2024] <<<<<< finish DM-102 >>>>>> rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:18:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:18:50 CST 2024] <<<<<< start DM-010 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:18:51 CST 2024] <<<<<< test case case_sensitive success! >>>>>> start running case: [checkpoint_transaction] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit wait process dm-master.test exit... alter: add column age int; 1.1.0 process dm-syncer.test already exit [Fri Apr 26 19:18:51 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:18:51 CST 2024] <<<<<< start DM-103 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8263 alive the 2-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:18:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8263 alive the 3-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8263 alive the 4-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/source1.yaml" alter: add key name (name); 1.1.0 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-task.yaml " wait for rpc addr 127.0.0.1:8263 alive the 5-th time [Fri Apr 26 19:18:54 CST 2024] <<<<<< finish DM-010 >>>>>> wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:55 CST 2024] <<<<<< finish DM-103 pessimistic >>>>>> [Fri Apr 26 19:18:55 CST 2024] <<<<<< start DM-103 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8263 alive the 6-th time alter: add key name (name); 1.1.0 [Fri Apr 26 19:18:55 CST 2024] <<<<<< start DM-011 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8263 alive the 7-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive start dm-master1 and dm-master2 again [Fri Apr 26 19:18:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:18:57 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" [Fri Apr 26 19:18:57 CST 2024] <<<<<< finish DM-011 >>>>>> kill dm-master1 wait process dm-master.test exit... alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 [Fri Apr 26 19:18:58 CST 2024] <<<<<< start DM-012 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:18:59 CST 2024] <<<<<< finish DM-103 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" wait process dm-master.test exit... [Fri Apr 26 19:19:00 CST 2024] <<<<<< start DM-104 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:19:00 CST 2024] <<<<<< finish DM-012 >>>>>> wait process dm-master.test exit... [Fri Apr 26 19:19:01 CST 2024] <<<<<< start DM-013 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" dmctl test cmd: "list-member --master" got=5 expected=5 [Fri Apr 26 19:19:01 CST 2024] <<<<<< start test_evict_leader >>>>>> leader is master4 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 14341 0 --:--:-- --:--:-- --:--:-- 16500 dmctl test cmd: "query-status test" got=2 expected=2 start check_worker_ungraceful_stop_with_retry times: 0 new leader is master3 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... leader is master3 dmctl test cmd: "operate-leader evict" alter: add column age int; pt-online-schema-change 3.0.12 alter: add column age int; % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 16113 0 --:--:-- --:--:-- --:--:-- 33000 new leader is master2 pt-online-schema-change 3.0.12 leader is master2 dmctl test cmd: "operate-leader evict" dmctl test cmd: "query-status test" got=1 expected=1 start check_worker_ungraceful_stop_with_retry times: 1 alter: add key name (name); dmctl test cmd: "query-status t-ร‹!s`t" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 15714 0 --:--:-- --:--:-- --:--:-- 16500 new leader is master1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:03 CST 2024] <<<<<< test case ha_cases_1 success! >>>>>> start running case: [ha_cases_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:03 CST 2024] <<<<<< start test_kill_master_in_sync >>>>>> [Fri Apr 26 19:19:03 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit pt-online-schema-change 3.0.12 leader is master1 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 16802 0 --:--:-- --:--:-- --:--:-- 33000 [Fri Apr 26 19:19:03 CST 2024] <<<<<< finish DM-013 >>>>>> clean source table alter: add key name (name); new leader is master5 got=2 expected=2 pt-online-schema-change 3.0.12 leader is master5 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 14932 0 --:--:-- --:--:-- --:--:-- 16500 cancel evict leader on master1, and master1 will be the leader start check_worker_ungraceful_stop_with_retry times: 2 check_worker_ungraceful_stop_with_retry success after retry: 2 after ungraceful exit data in source count: 31 data in tidb count: 17 ungraceful stop test success [Fri Apr 26 19:19:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task t-ร‹!s`t" dmctl test cmd: "stop-task test" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; dmctl test cmd: "operate-leader cancel-evict" cancel evict leader twice, and test cancel evict leader from http interface % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 19130 0 --:--:-- --:--:-- --:--:-- 33000 leader is master1 cancel evict leader on all masters cancel master port 8361 pt-online-schema-change 3.0.12 [Fri Apr 26 19:19:04 CST 2024] <<<<<< start DM-014 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" dmctl test cmd: "operate-leader cancel-evict" cancel master port 8461 dmctl test cmd: "operate-leader cancel-evict" cancel master port 8561 dmctl test cmd: "operate-leader cancel-evict" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; [Fri Apr 26 19:19:04 CST 2024] <<<<<< finish DM-104 optimistic >>>>>> cancel master port 8661 dmctl test cmd: "operate-leader cancel-evict" [Fri Apr 26 19:19:04 CST 2024] <<<<<< finish test_evict_leader >>>>>> [Fri Apr 26 19:19:04 CST 2024] <<<<<< start test_list_member_command >>>>>> pt-online-schema-change 3.0.12 wait for rpc addr 127.0.0.1:8261 alive the 2-th time alter: add column age int; pt-online-schema-change 3.0.12 import prepare data start DM worker and master cluster [Fri Apr 26 19:19:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:19:05 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:19:05 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:19:05 CST 2024] <<<<<< start DM-105 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" alter: add column age int; dmctl test cmd: "query-status test" got=1 expected=1 pt-online-schema-change 3.0.12 current leader is master1 dmctl test cmd: "list-member --master" got=5 expected=5 kill leader master1 run tidb sql failed 1-th time, retry later alter: add key name (name); pt-online-schema-change 3.0.12 rpc addr 127.0.0.1:8261 is alive alter: add key name (name); pt-online-schema-change 3.0.12 rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Fri Apr 26 19:19:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process master1 exit... dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; pt-online-schema-change 3.0.12 alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; pt-online-schema-change 3.0.12 wait process master1 exit... process master1 already exit dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" [Fri Apr 26 19:19:08 CST 2024] <<<<<< finish DM-014 >>>>>> use sync_diff_inspector to check increment data check diff successfully start dm-worker3 and kill dm-worker2 current leader is master2 dmctl test cmd: "list-member --master" got=4 expected=4 kill leader master2 [Fri Apr 26 19:19:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker2 exit... dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:09 CST 2024] <<<<<< finish DM-105 optimistic >>>>>> [Fri Apr 26 19:19:09 CST 2024] <<<<<< start DM-015 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:08 CST 2024] <<<<<< finish DM-4215 pessimistic >>>>>> [Fri Apr 26 19:19:08 CST 2024] <<<<<< start DM-4215 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive wait process master2 exit... got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" [Fri Apr 26 19:19:10 CST 2024] <<<<<< start DM-106 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:19:10 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time got=1 expected=2 command: query-status test "ErrScope": "downstream" count: 1 != expected: 2, failed the 0-th time, will retry again got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:11 CST 2024] <<<<<< finish DM-015 >>>>>> wait process master2 exit... process master2 already exit [Fri Apr 26 19:19:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8264 is alive wait and check task running dmctl test cmd: "query-status test" got=3 expected=3 start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " dmctl test cmd: "query-status test" check diff failed 2-th time, retry later rpc addr 127.0.0.1:8261 is alive got=2 expected=2 run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... [Fri Apr 26 19:19:12 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time [Fri Apr 26 19:19:12 CST 2024] <<<<<< start DM-016 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase rpc addr 127.0.0.1:8361 is alive got=3 expected=3 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later check diff failed 3-th time, retry later wait process dm-worker.test exit... use sync_diff_inspector to check increment2 data now! check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive current leader is master3 dmctl test cmd: "list-member --master" got=5 expected=5 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:14 CST 2024] <<<<<< finish DM-106 optimistic >>>>>> current leader is master3 kill master1 wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:15 CST 2024] <<<<<< finish DM-4215 optimistic >>>>>> dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:15 CST 2024] <<<<<< start DM-107 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check diff failed 4-th time, retry later wait process dm-master1 exit... [Fri Apr 26 19:19:15 CST 2024] <<<<<< finish DM-016 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:16 CST 2024] <<<<<< finish test_query_timeout >>>>>> [Fri Apr 26 19:19:16 CST 2024] <<<<<< start test_regexpr_router regexpr-task.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:16 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:19:16 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> use sync_diff_inspector to check increment data check diff successfully [Fri Apr 26 19:19:16 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Fri Apr 26 19:19:16 CST 2024] <<<<<< start DM-017 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-master1 exit... process dm-master1 already exit got=3 expected=3 wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:19:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master1 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 5-th time, retry later dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:5043 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:5548 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:19:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master1 exit... process dm-master1 already exit wait and check task running got=3 expected=3 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:19 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Fri Apr 26 19:19:19 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:19 CST 2024] <<<<<< test case online_ddl success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // timeout dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } check diff failed 6-th time, retry later [Pipeline] // podTemplate [Pipeline] } dmctl test cmd: "stop-task test" [Pipeline] // withEnv [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Pipeline] // stage [Pipeline] } check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:6850 alter table handle_error.tb1 add column c varchar(20);" [Fri Apr 26 19:19:20 CST 2024] <<<<<< finish DM-107 optimistic >>>>>> [Fri Apr 26 19:19:20 CST 2024] <<<<<< finish DM-017 >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:7521 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:21 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> [Fri Apr 26 19:19:21 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" [Fri Apr 26 19:19:21 CST 2024] <<<<<< start DM-108 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:19:21 CST 2024] <<<<<< start DM-018 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check diff failed 7-th time, retry later [Fri Apr 26 19:19:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member --master" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" got=5 expected=5 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:8915 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:9494 alter table handle_error.tb1 add column c varchar(20);" wait process dm-master.test exit... [Fri Apr 26 19:19:23 CST 2024] <<<<<< finish DM-018 >>>>>> current leader is master3 kill master2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:23 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Fri Apr 26 19:19:23 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:19:24 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check diff successfully pause task and check status dmctl test cmd: "pause-task test" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master2 exit... check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:10980 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" wait process dm-master2 exit... process dm-master2 already exit dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:11467 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:26 CST 2024] <<<<<< finish DM-019 >>>>>> [Fri Apr 26 19:19:26 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:19:26 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> wait process dm-worker.test exit... [Fri Apr 26 19:19:27 CST 2024] <<<<<< start DM-4219 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:27 CST 2024] <<<<<< finish test_regexpr_router regexpr-task.yaml >>>>>> [Fri Apr 26 19:19:27 CST 2024] <<<<<< start test_regexpr_router regexpr-task-lightning.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> check diff failed 3-th time, retry later [Fri Apr 26 19:19:27 CST 2024] <<<<<< finish DM-019 >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Fri Apr 26 19:19:28 CST 2024] <<<<<< start DM-020 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" run tidb sql failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:30 CST 2024] <<<<<< finish DM-108 optimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:19:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:19:30 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:19:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:19:31 CST 2024] <<<<<< start DM-109 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:31 CST 2024] <<<<<< finish DM-4219 pessimistic >>>>>> [Fri Apr 26 19:19:31 CST 2024] <<<<<< start DM-4219 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8361 is alive dmctl test cmd: "list-member --master" got=5 expected=5 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" current leader is master3 dmctl test cmd: "query-status test" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" current leader is master3 kill master4 got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:32 CST 2024] <<<<<< finish DM-020 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" wait process dm-master4 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" [Fri Apr 26 19:19:33 CST 2024] <<<<<< start DM-021 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" [Fri Apr 26 19:19:33 CST 2024] <<<<<< finish DM-4219 optimistic >>>>>> wait process dm-master4 exit... process dm-master4 already exit [Fri Apr 26 19:19:34 CST 2024] <<<<<< start DM-4220 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed 2-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" wait process dm-master.test exit... process dm-master.test already exit check diff failed 3-th time, retry later dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:16594" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:17605" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:16787" dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:37 CST 2024] <<<<<< finish DM-021 >>>>>> dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:17824" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:16787" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:17824" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test" [Fri Apr 26 19:19:38 CST 2024] <<<<<< start DM-022 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" run tidb sql failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:19:39 CST 2024] <<<<<< finish DM-109 optimistic >>>>>> dmctl test cmd: "query-status test" graceful pause test success start check pause diff check diff successfully resume task and check status dmctl test cmd: "resume-task test" [Fri Apr 26 19:19:39 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time dmctl test cmd: "query-status test" kill dm-worker1 dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:19:40 CST 2024] <<<<<< finish test_regexpr_router regexpr-task-lightning.yaml >>>>>> [Fri Apr 26 19:19:40 CST 2024] <<<<<< start DM-110 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:19:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:16787" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:17824" dmctl test cmd: "binlog revert test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:17824" rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "list-member --master" got=5 expected=5 wait process dm-worker1 exit... dmctl test cmd: "binlog revert test" dmctl test cmd: "stop-task test" current leader is master3 kill master5 rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:19:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:19:41 CST 2024] <<<<<< finish DM-4220 pessimistic >>>>>> [Fri Apr 26 19:19:41 CST 2024] <<<<<< start DM-4220 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:19:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master5 exit... check diff failed 1-th time, retry later [Fri Apr 26 19:19:42 CST 2024] <<<<<< finish DM-022 >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Fri Apr 26 19:19:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master5 exit... process dm-master5 already exit got=2 expected=2 dmctl test cmd: "binlog revert test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 stop task dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:43 CST 2024] <<<<<< start DM-023 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:19218" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:20496" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" got=2 expected=2 dmctl test cmd: "binlog revert test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:19411" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:20715" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:20715" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" run tidb sql failed 1-th time, retry later [Fri Apr 26 19:19:45 CST 2024] <<<<<< finish DM-023 >>>>>> dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task /tmp/dm_test/all_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" check diff failed 3-th time, retry later [Fri Apr 26 19:19:46 CST 2024] <<<<<< start DM-026 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:20715" dmctl test cmd: "binlog revert test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:20715" dmctl test cmd: "binlog revert test" dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" [Fri Apr 26 19:19:47 CST 2024] <<<<<< finish DM-4220 optimistic >>>>>> got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:48 CST 2024] <<<<<< finish DM-026 >>>>>> [Fri Apr 26 19:19:48 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time check diff successfully check diff failed at last dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:48 CST 2024] <<<<<< finish DM-110 optimistic >>>>>> [Fri Apr 26 19:19:48 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "pause-task t-ร‹!s`t" dmctl test cmd: "operate-schema set -s mysql-replica-01 t-ร‹!s`t -d all_mode -t no_diff /tmp/dm_test/all_mode/schema.sql" dmctl test cmd: "resume-task t-ร‹!s`t" [Fri Apr 26 19:19:49 CST 2024] <<<<<< start DM-027 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "list-member --master" got=5 expected=5 dmctl test cmd: "list-member --worker --name=worker1,worker2" [Fri Apr 26 19:19:49 CST 2024] <<<<<< start DM-111 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source1.yaml" wait pattern dm-worker1.toml exit... got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:21842" dmctl test cmd: "list-member --worker" wait pattern dm-worker1.toml exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:23387" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source2.yaml" wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Fri Apr 26 19:19:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" [Fri Apr 26 19:19:51 CST 2024] <<<<<< finish DM-027 >>>>>> dmctl test cmd: "list-member" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status t-ร‹!s`t" [Fri Apr 26 19:19:52 CST 2024] <<<<<< start DM-028 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" got=1 expected=1 wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Fri Apr 26 19:19:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "stop-task test" check diff failed 2-th time, retry later dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:53 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Fri Apr 26 19:19:53 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:19:53 CST 2024] <<<<<< finish DM-028 >>>>>> dmctl test cmd: "list-member --name=worker1,worker2" got=2 expected=2 kill worker1 rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:19:54 CST 2024] <<<<<< start DM-030 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:23811" check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:25545" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --name=worker1,worker2" got=1 expected=1 got=1 expected=1 kill worker2 [Fri Apr 26 19:19:56 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> [Fri Apr 26 19:19:56 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker2 exit... check diff failed at last dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:57 CST 2024] <<<<<< finish DM-111 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member" dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:26038" got=2 expected=2 [Fri Apr 26 19:19:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:27703" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:58 CST 2024] <<<<<< finish DM-030 pessimistic >>>>>> [Fri Apr 26 19:19:58 CST 2024] <<<<<< start DM-030 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "stop-task test" [Fri Apr 26 19:19:58 CST 2024] <<<<<< start DM-112 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:19:59 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Fri Apr 26 19:19:59 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:19:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff failed 1-th time, retry later [Fri Apr 26 19:20:00 CST 2024] <<<<<< finish DM-030 optimistic >>>>>> got=2 expected=2 graceful stop test success start check stop diff check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config master master1" dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:28265" wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "config master master2" got=1 expected=1 [Fri Apr 26 19:20:01 CST 2024] <<<<<< finish test_list_member_command >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:29861" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" [Fri Apr 26 19:20:01 CST 2024] <<<<<< start DM-031 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:20:02 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> check diff failed 2-th time, retry later got=2 expected=2 start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" process dm-syncer.test already exit [Fri Apr 26 19:20:03 CST 2024] <<<<<< test case checkpoint_transaction success! >>>>>> start running case: [check_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:20:03 CST 2024] <<<<<< start DM-4201 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/single-source-no-sharding.yaml --remove-meta" [Fri Apr 26 19:20:03 CST 2024] <<<<<< finish DM-031 pessimistic >>>>>> [Fri Apr 26 19:20:03 CST 2024] <<<<<< start DM-031 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file kill dm-master1 and kill dm-master2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:30234" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff failed 3-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:20:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master1 exit... [Fri Apr 26 19:20:05 CST 2024] <<<<<< finish DM-031 optimistic >>>>>> [Fri Apr 26 19:20:05 CST 2024] <<<<<< finish DM-4201 >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/source1.yaml" wait process dm-master1 exit... process dm-master1 already exit [Fri Apr 26 19:20:06 CST 2024] <<<<<< start DM-032 optimistic >>>>>> after restart dm-worker, task should resume automatically dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml" HTTP 127.0.0.1:8261/apis/v1alpha1/status/t-ร‹!s`t is alive 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-noshard.yaml" check diff failed at last dmctl test cmd: "stop-task test" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-sharding.yaml" [Fri Apr 26 19:20:07 CST 2024] <<<<<< finish DM-112 optimistic >>>>>> wait process dm-master2 exit... --> start test_privileges_can_migrate... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:20:08 CST 2024] <<<<<< start DM-113 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master2 exit... process dm-master2 already exit wait and check task running HTTP 127.0.0.1:8461/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 [Fri Apr 26 19:20:08 CST 2024] <<<<<< finish DM-032 optimistic >>>>>> wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process tidb-server exit... process tidb-server already exit [Fri Apr 26 19:20:09 CST 2024] <<<<<< start DM-033 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" pass test_privileges_can_migrate --> start test_privilege_precheck... dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" wait process dm-worker.test exit... dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" pass test_privilege_precheck 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive use sync_diff_inspector to check data now! check diff successfully dmctl test cmd: "offline-member --master --name master1" dmctl test cmd: "offline-member --master --name master2" kill dm-master3 dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:11 CST 2024] <<<<<< test case handle_error_2 success! >>>>>> start running case: [handle_error_3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:12 CST 2024] <<<<<< finish DM-113 pessimistic >>>>>> [Fri Apr 26 19:20:12 CST 2024] <<<<<< start DM-113 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master3 exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:20:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-master3 exit... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:14 CST 2024] <<<<<< finish DM-033 pessimistic >>>>>> got=2 expected=2 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source1.yaml" wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-master3 exit... process dm-master3 already exit [Fri Apr 26 19:20:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:20:15 CST 2024] <<<<<< start DM-034 pessimistic >>>>>> wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source2.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start running case: [dm_syncer] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/run.sh... Verbose mode = false check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:16 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later got=3 expected=3 [Fri Apr 26 19:20:16 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Fri Apr 26 19:20:16 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" [Fri Apr 26 19:20:16 CST 2024] <<<<<< finish DM-113 optimistic >>>>>> got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:17 CST 2024] <<<<<< finish DM-034 pessimistic >>>>>> [Fri Apr 26 19:20:17 CST 2024] <<<<<< start DM-034 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:20:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:20:17 CST 2024] <<<<<< start DM-114 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8461 is alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 2-th time, retry later dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:1559 alter table handle_error.tb1 add column d varchar(20);" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:20:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:1735 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:20:19 CST 2024] <<<<<< finish DM-034 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dm_syncer/source1.yaml" check diff failed 1-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/dm_syncer/source2.yaml" check diff successfully dmctl test cmd: "pause-relay -s mysql-replica-01" [Fri Apr 26 19:20:20 CST 2024] <<<<<< start DM-035 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "resume-relay -s mysql-replica-01" dmctl test cmd: "start-task /tmp/dm_test/dm_syncer/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:21 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Fri Apr 26 19:20:21 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" run tidb sql failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:22 CST 2024] <<<<<< finish DM-114 pessimistic >>>>>> [Fri Apr 26 19:20:22 CST 2024] <<<<<< start DM-114 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" relay logs dm-it-7a56601b-b3dc-4e67-9c90-d47b7c9e0bc2-7nb2f-hnglk-bin.000002 relay.meta check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/all_mode/worker2/dumped_data.t-ร‹!s`t: No such file or directory worker2 auto removed dump files check no password in log dmctl test cmd: "query-status t-ร‹!s`t" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" [Fri Apr 26 19:20:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" matched matched [Fri Apr 26 19:20:23 CST 2024] <<<<<< start test_source_and_target_with_empty_gtid >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:3856 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:4247 alter table handle_error.tb1 add column d varchar(20);" got=2 expected=2 wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "pause-task test" dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" got=3 expected=3 got=1 expected=1 got=1 expected=1 4 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Fri Apr 26 19:20:24 CST 2024] <<<<<< finish DM-035 optimistic >>>>>> [Fri Apr 26 19:20:24 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> [Fri Apr 26 19:20:24 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/bin/dm-syncer.test --log-file=/tmp/dm_test/dm_syncer/syncer1/log/dm-syncer.log --config=/tmp/dm_test/dm_syncer/dm-syncer-1.toml >> /tmp/dm_test/dm_syncer/syncer1/log/stdout.log 2>&1 & /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/bin/dm-syncer.test --log-file=/tmp/dm_test/dm_syncer/syncer2/log/dm-syncer.log --config=/tmp/dm_test/dm_syncer/dm-syncer-2.toml --meta=/tmp/dm_test/dm_syncer/old_meta_file --syncer-config-format >> /tmp/dm_test/dm_syncer/syncer2/log/stdout.log 2>&1 & wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... wait process dm-master.test exit... [Fri Apr 26 19:20:26 CST 2024] <<<<<< finish DM-114 optimistic >>>>>> dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:6411 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:6759 alter table handle_error.tb1 add column d varchar(20);" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Fri Apr 26 19:20:27 CST 2024] <<<<<< start DM-115 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:20:27 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Fri Apr 26 19:20:27 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:8966 alter table handle_error.tb1 add column d varchar(20);" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:9271 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:30 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:30 CST 2024] <<<<<< test case shardddl1_1 success! >>>>>> start running case: [shardddl2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:20:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/all_mode/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Fri Apr 26 19:20:31 CST 2024] <<<<<< start DM-4210 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 gtid is empty start task and check stage dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task-no-gtid.yaml --remove-meta=true" wait process dm-master.test exit... dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:20:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:11239 alter table handle_error.tb1 add column d int;alter table handle_error.tb1 add unique(d);" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column e int unique;" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff successfully ERROR 1146 (42S02) at line 1: Table 'all_mode.t2' doesn't exist run tidb sql failed 1-th time, retry later [Fri Apr 26 19:20:33 CST 2024] <<<<<< finish DM-115 optimistic >>>>>> got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source1.yaml" [Fri Apr 26 19:20:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... [Fri Apr 26 19:20:34 CST 2024] <<<<<< start DM-116 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully check diff successfully check diff successfully 1 dm-master alive 0 dm-worker alive 2 dm-syncer alive wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source2.yaml" dmctl test cmd: "stop-task test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Fri Apr 26 19:20:36 CST 2024] <<<<<< finish DM-4210 >>>>>> [Fri Apr 26 19:20:36 CST 2024] <<<<<< start DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:36 CST 2024] <<<<<< test case dm_syncer success! >>>>>> start running case: [downstream_diff_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:20:37 CST 2024] <<<<<< start DM-4193 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker 1 wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:20:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:38 CST 2024] <<<<<< finish DM-116 pessimistic >>>>>> [Fri Apr 26 19:20:38 CST 2024] <<<<<< start DM-116 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 wait process dm-worker1 exit... dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:13027" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:12152" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source1.yaml" [Fri Apr 26 19:20:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:13027" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:12152" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:13413" wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:20:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> dmctl test cmd: "binlog revert test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:13413" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:13413" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:12590" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source2.yaml" [Fri Apr 26 19:20:40 CST 2024] <<<<<< finish DM-116 optimistic >>>>>> [Fri Apr 26 19:20:40 CST 2024] <<<<<< finish DM-4193 pessimistic >>>>>> [Fri Apr 26 19:20:40 CST 2024] <<<<<< start DM-4193 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive check log contain failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-task.yaml --remove-meta" [Fri Apr 26 19:20:41 CST 2024] <<<<<< start DM-117 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:15189" run tidb sql failed 1-th time, retry later dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:14529" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:15189" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:14529" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl3_1/schema.sql -s mysql-replica-01" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:15575" dmctl test cmd: "binlog revert test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:15575" dmctl test cmd: "query-status test" got=1 expected=1 restart dm-master dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:15575" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:14967" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-master exit... [Fri Apr 26 19:20:44 CST 2024] <<<<<< finish DM-117 optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Fri Apr 26 19:20:45 CST 2024] <<<<<< start DM-118 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:46 CST 2024] <<<<<< finish DM-4193 optimistic >>>>>> wait process dm-master.test exit... [Fri Apr 26 19:20:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:20:47 CST 2024] <<<<<< start DM-4230 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column d int unique;" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:49 CST 2024] <<<<<< finish DM-118 pessimistic >>>>>> [Fri Apr 26 19:20:49 CST 2024] <<<<<< start DM-118 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog revert test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 restart dm-worker 1 wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:50 CST 2024] <<<<<< test case downstream_diff_index success! >>>>>> start running case: [downstream_more_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:20:50 CST 2024] <<<<<< finish DM-4230 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker1 exit... wait process dm-master.test exit... check diff failed 1-th time, retry later [Fri Apr 26 19:20:51 CST 2024] <<<<<< start DM-4177 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:20:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:20:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/downstream_more_column/source1.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip wrong-task" dmctl test cmd: "binlog skip" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "binlog skip test -b dm-it-3f0cadde-a643-4033-a34c-a2527e3151db-85trz-jlhpl-bin|000001.000001:19397 -s wrong-source" dmctl test cmd: "binlog skip test -b mysql-bin|1111 -s wrong-source" wait process dm-master.test exit... check diff failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" [Fri Apr 26 19:20:53 CST 2024] <<<<<< finish DM-118 optimistic >>>>>> dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int;"" dmctl test cmd: "binlog skip test -b mysql-bin.000000:00000" dmctl test cmd: "binlog aaa test" wait process dm-master.test exit... dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Fri Apr 26 19:20:54 CST 2024] <<<<<< finish DM-4177 >>>>>> dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int" dmctl test cmd: "binlog revert test" run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... [Fri Apr 26 19:20:55 CST 2024] <<<<<< start DM-4231 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:55 CST 2024] <<<<<< finish DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> [Fri Apr 26 19:20:55 CST 2024] <<<<<< start DM-INIT_SCHEMA optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Fri Apr 26 19:20:44 CST 2024] <<<<<< finish test_kill_master_in_sync >>>>>> [Fri Apr 26 19:20:44 CST 2024] <<<<<< start test_kill_worker_in_sync >>>>>> [Fri Apr 26 19:20:44 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master cluster [Fri Apr 26 19:20:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:20:55 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:20:55 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int; alter table handle_error.tb1 add column d int unique;"" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/downstream_more_column/dm-task-incremental.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... dmctl test cmd: "stop-task test" [Fri Apr 26 19:20:57 CST 2024] <<<<<< finish DM-4231 >>>>>> dmctl test cmd: "query-status test" wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d downstream_more_column1 -t t1 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/data/schema.sql" dmctl test cmd: "resume-task test" wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:59 CST 2024] <<<<<< test case shardddl3_1 success! >>>>>> start running case: [shardddl4] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:20:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time restart dm-master rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Fri Apr 26 19:20:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" [Fri Apr 26 19:21:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:21:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source1.yaml" wait process dm-worker.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... [Fri Apr 26 19:21:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:21:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:21:03 CST 2024] <<<<<< test case downstream_more_column success! >>>>>> start running case: [drop_column_with_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:21:03 CST 2024] <<<<<< test case handle_error_3 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Fri Apr 26 19:21:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source2.yaml" [Pipeline] } [Pipeline] // timeout [Pipeline] } wait process dm-master.test exit... [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Fri Apr 26 19:21:05 CST 2024] <<<<<< start DM-119 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:21:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/drop_column_with_index/source1.yaml" [Fri Apr 26 19:21:07 CST 2024] <<<<<< finish DM-INIT_SCHEMA optimistic >>>>>> [Fri Apr 26 19:21:07 CST 2024] <<<<<< start DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-master.test exit... use sync_diff_inspector to check increment data check diff successfully [Fri Apr 26 19:21:08 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source kill dm-worker1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker 2 wait process dm-worker1 exit... check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker2 exit... check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" wait process dm-worker1 exit... process dm-worker1 already exit start worker3 [Fri Apr 26 19:21:10 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:21:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 3-th time, retry later rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker3" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later got=2 expected=2 kill dm-worker2 wait process dm-master.test exit... check diff failed 4-th time, retry later wait process dm-master.test exit... wait process dm-worker2 exit... dmctl test cmd: "query-status test" got=1 expected=1 restart dm-master wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit start worker4 [Fri Apr 26 19:21:14 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait process dm-master exit... dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... rpc addr 127.0.0.1:18262 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive query-status from all dm-master dmctl test cmd: "query-status test" check diff failed 5-th time, retry later got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" wait process dm-master exit... process dm-master already exit got=3 expected=3 wait process dm-master.test exit... check diff failed 6-th time, retry later wait process dm-master.test exit... got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Fri Apr 26 19:21:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... check diff failed 7-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... check diff failed 8-th time, retry later rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 restart dm-worker 2 wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker2 exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:21:22 CST 2024] <<<<<< test case drop_column_with_index success! >>>>>> start running case: [duplicate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... [Fri Apr 26 19:21:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit [Fri Apr 26 19:21:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:21:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 9-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" wait process dm-master.test exit... got=3 expected=3 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:21:25 CST 2024] <<<<<< test case ha_master success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } check diff failed 10-th time, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 2-th time, retry later check log contain failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" [Fri Apr 26 19:21:28 CST 2024] <<<<<< finish DM-119 pessimistic >>>>>> [Fri Apr 26 19:21:28 CST 2024] <<<<<< start DM-119 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:21:29 CST 2024] <<<<<< finish DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> [Fri Apr 26 19:21:29 CST 2024] <<<<<< start DM-RECOVER_LOCK optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later check log contain failed 3-th time, retry later check diff failed 2-th time, retry later check log contain failed 1-th time, retry later check log contain failed 4-th time, retry later check diff failed 3-th time, retry later restart dm-master check log contain failed 5-th time, retry later wait process dm-master exit... check diff failed 4-th time, retry later wait process dm-master exit... process dm-master already exit check log contain failed 6-th time, retry later check diff failed 5-th time, retry later [Fri Apr 26 19:21:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 7-th time, retry later check diff failed 6-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 8-th time, retry later rpc addr 127.0.0.1:8261 is alive restart dm-master check diff failed 7-th time, retry later wait process dm-master exit... check log contain failed 9-th time, retry later wait process dm-master exit... process dm-master already exit check diff failed 8-th time, retry later check log contain failed 10-th time, retry later [Fri Apr 26 19:21:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 9-th time, retry later check log contain failed 11-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff failed 10-th time, retry later rpc addr 127.0.0.1:8261 is alive check diff successfully dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:21:49 CST 2024] <<<<<< finish DM-RECOVER_LOCK optimistic >>>>>> run DM_DropAddColumn case #0 [Fri Apr 26 19:21:49 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 12-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" [Fri Apr 26 19:21:50 CST 2024] <<<<<< finish DM-119 optimistic >>>>>> got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later check log contain failed 13-th time, retry later [Fri Apr 26 19:21:51 CST 2024] <<<<<< start DM-120 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 14-th time, retry later check diff failed 1-th time, retry later dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check log contain failed 15-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:21:56 CST 2024] <<<<<< finish DM-120 pessimistic >>>>>> [Fri Apr 26 19:21:56 CST 2024] <<<<<< start DM-120 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 16-th time, retry later check diff failed 3-th time, retry later check log contain failed 17-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:22:00 CST 2024] <<<<<< finish DM-120 optimistic >>>>>> dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:22:01 CST 2024] <<<<<< start DM-121 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" [Fri Apr 26 19:22:01 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #1 [Fri Apr 26 19:22:01 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 18-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully restart dm-master check diff failed 1-th time, retry later check log contain failed 19-th time, retry later wait process dm-master exit... wait process dm-master exit... process dm-master already exit check diff failed 2-th time, retry later check log contain failed 20-th time, retry later [Fri Apr 26 19:22:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 3-th time, retry later check log contain failed 21-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff failed 4-th time, retry later rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later check log contain failed 22-th time, retry later dmctl test cmd: "shard-ddl-lock" check diff failed 5-th time, retry later check log contain failed 23-th time, retry later got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 <<<<<< test_source_and_target_with_empty_gtid success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff failed 6-th time, retry later check log contain failed 24-th time, retry later wait process dm-worker.test exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... check diff failed 7-th time, retry later check log contain failed 25-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:22:16 CST 2024] <<<<<< test case all_mode success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } check diff failed 3-th time, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check log contain failed 26-th time, retry later check diff failed 8-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:22:19 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #2 [Fri Apr 26 19:22:19 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff failed 9-th time, retry later check log contain failed 27-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully restart dm-master check diff failed 10-th time, retry later check log contain failed 28-th time, retry later wait process dm-master exit... wait process dm-master exit... process dm-master already exit check diff failed at last dmctl test cmd: "stop-task test" check log contain failed 29-th time, retry later [Fri Apr 26 19:22:24 CST 2024] <<<<<< finish DM-121 pessimistic >>>>>> [Fri Apr 26 19:22:24 CST 2024] <<<<<< start DM-121 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:22:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 30-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" check diff failed 2-th time, retry later got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check diff successfully [Fri Apr 26 19:22:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 3-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" check diff failed 2-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" check diff failed 4-th time, retry later check diff failed 3-th time, retry later check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" check diff failed 5-th time, retry later got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 check diff failed 1-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:22:35 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #3 [Fri Apr 26 19:22:35 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff failed 6-th time, retry later check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Fri Apr 26 19:22:36 CST 2024] <<<<<< finish test_kill_worker_in_sync >>>>>> 3 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... check diff failed 7-th time, retry later check diff failed 3-th time, retry later wait process dm-master.test exit... dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master wait process dm-master.test exit... check diff failed 8-th time, retry later wait process dm-master exit... check diff failed 4-th time, retry later wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit wait process dm-master.test exit... check diff failed 5-th time, retry later check diff failed 9-th time, retry later wait process dm-master.test exit... [Fri Apr 26 19:22:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... check diff failed 10-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:22:45 CST 2024] <<<<<< test case ha_cases_2 success! >>>>>> start running case: [ha_cases2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:22:45 CST 2024] <<<<<< start test_pause_task >>>>>> [Fri Apr 26 19:22:45 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check diff failed 6-th time, retry later clean source table rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" [Fri Apr 26 19:22:47 CST 2024] <<<<<< finish DM-121 optimistic >>>>>> check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive import prepare data start DM worker and master [Fri Apr 26 19:22:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:22:47 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:22:47 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:22:48 CST 2024] <<<<<< start DM-122 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... check diff failed 2-th time, retry later rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Fri Apr 26 19:22:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml" [Fri Apr 26 19:22:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... check diff failed 3-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:22:52 CST 2024] <<<<<< finish DM-122 pessimistic >>>>>> [Fri Apr 26 19:22:52 CST 2024] <<<<<< start DM-122 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:22:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:22:52 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:22:53 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #4 [Fri Apr 26 19:22:53 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8264 is alive [Fri Apr 26 19:22:53 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:22:54 CST 2024] <<<<<< finish DM-122 optimistic >>>>>> [Fri Apr 26 19:22:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:18262 is alive [Fri Apr 26 19:22:54 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" [Fri Apr 26 19:22:55 CST 2024] <<<<<< start DM-123 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check log contain failed 1-th time, retry later check diff failed 1-th time, retry later got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master wait process dm-master exit... check log contain failed 2-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:22:59 CST 2024] <<<<<< finish DM-123 pessimistic >>>>>> [Fri Apr 26 19:22:59 CST 2024] <<<<<< start DM-123 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master exit... process dm-master already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 3-th time, retry later [Fri Apr 26 19:23:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time use sync_diff_inspector to check increment data check diff successfully check diff successfully [Fri Apr 26 19:23:02 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully dmctl test cmd: "stop-task test" check log contain failed 4-th time, retry later [Fri Apr 26 19:23:03 CST 2024] <<<<<< finish DM-123 optimistic >>>>>> got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later [Fri Apr 26 19:23:04 CST 2024] <<<<<< start DM-124 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 5-th time, retry later got=2 expected=2 start dumping SQLs into source pause tasks test dmctl test cmd: "pause-task test" dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" pause tasks test2 dmctl test cmd: "pause-task test2" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "pause-task test2" check diff successfully dmctl test cmd: "stop-task test" check diff failed 2-th time, retry later dmctl test cmd: "query-status test2" [Fri Apr 26 19:23:06 CST 2024] <<<<<< finish DM-124 optimistic >>>>>> check log contain failed 6-th time, retry later [Fri Apr 26 19:23:07 CST 2024] <<<<<< start DM-125 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" resume tasks test dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "query-status test" resume tasks test2 dmctl test cmd: "resume-task test2" got=3 expected=3 dmctl test cmd: "resume-task test2" check log contain failed 7-th time, retry later got=3 expected=3 dmctl test cmd: "query-status test2" check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 8-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:11 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #5 [Fri Apr 26 19:23:11 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:23:11 CST 2024] <<<<<< finish DM-125 optimistic >>>>>> check log contain failed 9-th time, retry later [Fri Apr 26 19:23:12 CST 2024] <<<<<< start DM-126 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 10-th time, retry later dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 restart dm-master check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" wait process dm-master exit... [Fri Apr 26 19:23:16 CST 2024] <<<<<< finish DM-126 optimistic >>>>>> check log contain failed 11-th time, retry later wait process dm-master exit... process dm-master already exit [Fri Apr 26 19:23:17 CST 2024] <<<<<< start DM-127 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 12-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Fri Apr 26 19:23:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 13-th time, retry later check diff successfully wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:22 CST 2024] <<<<<< finish DM-127 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check log contain failed 14-th time, retry later [Fri Apr 26 19:23:23 CST 2024] <<<<<< start DM-128 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 15-th time, retry later check diff failed 2-th time, retry later check diff successfully dmctl test cmd: "stop-task test" check diff failed 3-th time, retry later [Fri Apr 26 19:23:27 CST 2024] <<<<<< finish DM-128 pessimistic >>>>>> [Fri Apr 26 19:23:27 CST 2024] <<<<<< start DM-128 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 16-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" [Fri Apr 26 19:23:29 CST 2024] <<<<<< finish DM-128 optimistic >>>>>> dmctl test cmd: "resume-task test" check log contain failed 17-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:29 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> [Fri Apr 26 19:23:29 CST 2024] <<<<<< start DM-ADD_DROP_PARTITIONS optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:23:30 CST 2024] <<<<<< start DM-129 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 18-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 19-th time, retry later run tidb sql failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:34 CST 2024] <<<<<< finish DM-129 pessimistic >>>>>> [Fri Apr 26 19:23:34 CST 2024] <<<<<< start DM-129 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 20-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:35 CST 2024] <<<<<< finish DM-ADD_DROP_PARTITIONS optimistic >>>>>> [Fri Apr 26 19:23:35 CST 2024] <<<<<< start DM-036 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:36 CST 2024] <<<<<< finish DM-129 optimistic >>>>>> check log contain failed 21-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later [Fri Apr 26 19:23:37 CST 2024] <<<<<< start DM-130 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:39 CST 2024] <<<<<< finish DM-130 pessimistic >>>>>> [Fri Apr 26 19:23:39 CST 2024] <<<<<< start DM-130 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 22-th time, retry later dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:39 CST 2024] <<<<<< finish DM-036 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later [Fri Apr 26 19:23:40 CST 2024] <<<<<< start DM-037 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 23-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:42 CST 2024] <<<<<< finish DM-037 pessimistic >>>>>> [Fri Apr 26 19:23:42 CST 2024] <<<<<< start DM-037 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb1 --action skip" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb2 --action skip" check log contain failed 24-th time, retry later dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:43 CST 2024] <<<<<< finish DM-130 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:44 CST 2024] <<<<<< finish DM-037 optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check log contain failed 25-th time, retry later [Fri Apr 26 19:23:45 CST 2024] <<<<<< start DM-038 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" check log contain failed 26-th time, retry later got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check log contain failed 27-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:23:49 CST 2024] <<<<<< test case shardddl4 success! >>>>>> start running case: [shardddl4_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:23:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:23:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 28-th time, retry later check diff failed 2-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source1.yaml" check log contain failed 29-th time, retry later [Fri Apr 26 19:23:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source2.yaml" check diff failed 3-th time, retry later check log contain failed 30-th time, retry later [Fri Apr 26 19:23:55 CST 2024] <<<<<< start DM-TABLE_CHECKPOINT_BACKWARD optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" check diff failed at last dmctl test cmd: "stop-task test" [Fri Apr 26 19:23:56 CST 2024] <<<<<< finish DM-038 pessimistic >>>>>> [Fri Apr 26 19:23:56 CST 2024] <<<<<< start DM-038 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully [Fri Apr 26 19:23:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" check diff failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" check diff failed 2-th time, retry later got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... check diff failed 3-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check diff failed at last dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:07 CST 2024] <<<<<< finish DM-038 optimistic >>>>>> wait process dm-worker.test exit... [Fri Apr 26 19:24:08 CST 2024] <<<<<< start DM-039 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:24:09 CST 2024] <<<<<< test case duplicate_event success! >>>>>> start running case: [expression_filter] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:24:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:24:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:12 CST 2024] <<<<<< finish DM-039 pessimistic >>>>>> [Fri Apr 26 19:24:12 CST 2024] <<<<<< start DM-039 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml " dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again [Fri Apr 26 19:24:14 CST 2024] <<<<<< finish DM-039 optimistic >>>>>> [Fri Apr 26 19:24:15 CST 2024] <<<<<< start DM-040 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" ls: cannot access /tmp/dm_test/expression_filter/worker1/schema-tracker*: No such file or directory schema tracker path has been cleaned 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:17 CST 2024] <<<<<< finish DM-040 pessimistic >>>>>> [Fri Apr 26 19:24:17 CST 2024] <<<<<< start DM-040 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" check diff failed 2-th time, retry later wait process dm-worker.test exit... [Fri Apr 26 19:24:19 CST 2024] <<<<<< finish DM-040 optimistic >>>>>> [Fri Apr 26 19:24:20 CST 2024] <<<<<< start DM-041 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff failed 3-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:24:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:24:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 4-th time, retry later check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task.yaml " [Fri Apr 26 19:24:24 CST 2024] <<<<<< finish DM-041 pessimistic >>>>>> [Fri Apr 26 19:24:24 CST 2024] <<<<<< start DM-041 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:25 CST 2024] <<<<<< finish DM-TABLE_CHECKPOINT_BACKWARD optimistic >>>>>> [Fri Apr 26 19:24:25 CST 2024] <<<<<< start DM-RESYNC_NOT_FLUSHED optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:26 CST 2024] <<<<<< finish DM-041 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker1 [Fri Apr 26 19:24:27 CST 2024] <<<<<< start DM-043 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" wait process worker1 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:29 CST 2024] <<<<<< finish DM-043 pessimistic >>>>>> [Fri Apr 26 19:24:29 CST 2024] <<<<<< start DM-043 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:24:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive restart dm-worker2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:31 CST 2024] <<<<<< finish DM-043 optimistic >>>>>> wait process worker2 exit... wait process dm-master.test exit... wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:24:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:24:35 CST 2024] <<<<<< test case expression_filter success! >>>>>> start running case: [extend_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: sql [Fri Apr 26 19:24:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:24:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:24:37 CST 2024] <<<<<< test case shardddl2 success! >>>>>> start running case: [shardddl2_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:24:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:24:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:24:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2_1/source1.yaml" [Fri Apr 26 19:24:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2_1/source2.yaml" run tidb sql failed 1-th time, retry later [Fri Apr 26 19:24:42 CST 2024] <<<<<< start DM-046 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:44 CST 2024] <<<<<< finish DM-046 pessimistic >>>>>> [Fri Apr 26 19:24:44 CST 2024] <<<<<< start DM-046 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... use sync_diff_inspector to check increment data check diff successfully check diff successfully [Fri Apr 26 19:24:32 CST 2024] <<<<<< finish test_pause_task >>>>>> [Fri Apr 26 19:24:32 CST 2024] <<<<<< start test_multi_task_reduce_and_restart_worker >>>>>> [Fri Apr 26 19:24:32 CST 2024] <<<<<< start test_multi_task_running >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master [Fri Apr 26 19:24:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:24:44 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:24:44 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:24:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:24:46 CST 2024] <<<<<< finish DM-046 optimistic >>>>>> wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Fri Apr 26 19:24:47 CST 2024] <<<<<< start DM-047 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Fri Apr 26 19:24:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: loader [Fri Apr 26 19:24:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:24:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:24:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:51 CST 2024] <<<<<< finish DM-047 pessimistic >>>>>> [Fri Apr 26 19:24:51 CST 2024] <<<<<< start DM-047 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:24:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 [Fri Apr 26 19:24:52 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:53 CST 2024] <<<<<< finish DM-047 optimistic >>>>>> rpc addr 127.0.0.1:8264 is alive [Fri Apr 26 19:24:53 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time [Fri Apr 26 19:24:54 CST 2024] <<<<<< start DM-048 pessimistic >>>>>> dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:18262 is alive [Fri Apr 26 19:24:54 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:24:58 CST 2024] <<<<<< finish DM-048 pessimistic >>>>>> [Fri Apr 26 19:24:58 CST 2024] <<<<<< start DM-048 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:00 CST 2024] <<<<<< finish DM-048 optimistic >>>>>> wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:25:01 CST 2024] <<<<<< start DM-049 pessimistic >>>>>> wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 use sync_diff_inspector to check increment data check diff successfully check diff successfully wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:25:02 CST 2024] <<<<<< test case extend_column success! >>>>>> start running case: [fake_rotate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:25:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:25:03 CST 2024] <<<<<< finish test_multi_task_running >>>>>> start dumping SQLs into source "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) restart unuse worker3 try to kill worker port 8264 wait process dm-worker3 exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:25:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker3 exit... process dm-worker3 already exit dmctl test cmd: "list-member --worker --name=worker3" got=1 expected=1 start dm-worker3 [Fri Apr 26 19:25:05 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/fake_rotate_event/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-task.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:06 CST 2024] <<<<<< finish DM-049 pessimistic >>>>>> [Fri Apr 26 19:25:06 CST 2024] <<<<<< start DM-049 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8264 is alive restart unuse worker4 try to kill worker port 18262 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check diff successfully kill dm-worker wait process dm-worker4 exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker1 exit... check diff successfully restart dm-worker1 wait process worker1 exit... wait process dm-worker4 exit... process dm-worker4 already exit dmctl test cmd: "list-member --worker --name=worker4" got=1 expected=1 start dm-worker4 [Fri Apr 26 19:25:08 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:25:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:25:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:18262 is alive restart unuse worker5 try to kill worker port 18263 rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive restart dm-worker2 got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker5 exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:11 CST 2024] <<<<<< finish DM-049 optimistic >>>>>> wait process worker2 exit... wait process dm-master.test exit... wait process dm-worker5 exit... process dm-worker5 already exit dmctl test cmd: "list-member --worker --name=worker5" got=1 expected=1 start dm-worker5 [Fri Apr 26 19:25:11 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:25:12 CST 2024] <<<<<< start DM-050 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:25:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:18263 is alive try to kill worker port 8262 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-task test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:13 CST 2024] <<<<<< finish DM-RESYNC_NOT_FLUSHED optimistic >>>>>> [Fri Apr 26 19:25:13 CST 2024] <<<<<< start DM-RESYNC_TXN_INTERRUPT optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:25:13 CST 2024] <<<<<< finish DM-050 pessimistic >>>>>> [Fri Apr 26 19:25:13 CST 2024] <<<<<< start DM-050 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker1 exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker1 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker1 exit... process dm-worker1 already exit worker1 was killed dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:25:15 CST 2024] <<<<<< test case fake_rotate_event success! >>>>>> start running case: [foreign_key] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:25:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:15 CST 2024] <<<<<< finish DM-050 optimistic >>>>>> wait process worker1 exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Fri Apr 26 19:25:16 CST 2024] <<<<<< start DM-051 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:25:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:25:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive restart dm-worker2 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/foreign_key/source1.yaml" [Fri Apr 26 19:25:18 CST 2024] <<<<<< finish DM-051 pessimistic >>>>>> [Fri Apr 26 19:25:18 CST 2024] <<<<<< start DM-051 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-task.yaml --remove-meta" wait process worker2 exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:25:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:20 CST 2024] <<<<<< finish DM-051 optimistic >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive begin; insert into shardddl1.tb2 values (1,1); insert into shardddl1.tb2 values (2,2); insert into shardddl1.tb2 values (3,3); insert into shardddl1.tb2 values (4,4); insert into shardddl1.tb2 values (5,5); insert into shardddl1.tb2 values (6,6); insert into shardddl1.tb2 values (7,7); insert into shardddl1.tb2 values (8,8); insert into shardddl1.tb2 values (9,9); insert into shardddl1.tb2 values (10,10); commit; begin; insert into shardddl1.t_1 values (11,11); insert into shardddl1.t_1 values (12,12); insert into shardddl1.t_1 values (13,13); insert into shardddl1.t_1 values (14,14); insert into shardddl1.t_1 values (15,15); insert into shardddl1.t_1 values (16,16); insert into shardddl1.t_1 values (17,17); insert into shardddl1.t_1 values (18,18); insert into shardddl1.t_1 values (19,19); insert into shardddl1.t_1 values (20,20); insert into shardddl1.t_1 values (21,21); insert into shardddl1.t_1 values (22,22); insert into shardddl1.t_1 values (23,23); insert into shardddl1.t_1 values (24,24); insert into shardddl1.t_1 values (25,25); insert into shardddl1.t_1 values (26,26); insert into shardddl1.t_1 values (27,27); insert into shardddl1.t_1 values (28,28); insert into shardddl1.t_1 values (29,29); insert into shardddl1.t_1 values (30,30); insert into shardddl1.t_1 values (31,31); insert into shardddl1.t_1 values (32,32); insert into shardddl1.t_1 values (33,33); insert into shardddl1.t_1 values (34,34); insert into shardddl1.t_1 values (35,35); insert into shardddl1.t_1 values (36,36); insert into shardddl1.t_1 values (37,37); insert into shardddl1.t_1 values (38,38); insert into shardddl1.t_1 values (39,39); insert into shardddl1.t_1 values (40,40); insert into shardddl1.t_1 values (41,41); insert into shardddl1.t_1 values (42,42); insert into shardddl1.t_1 values (43,43); insert into shardddl1.t_1 values (44,44); insert into shardddl1.t_1 values (45,45); insert into shardddl1.t_1 values (46,46); insert into shardddl1.t_1 values (47,47); insert into shardddl1.t_1 values (48,48); insert into shardddl1.t_1 values (49,49); insert into shardddl1.t_1 values (50,50); commit; begin; insert into shardddl1.tb1 values (51,51); insert into shardddl1.tb1 values (52,52); insert into shardddl1.tb1 values (53,53); insert into shardddl1.tb1 values (54,54); insert into shardddl1.tb1 values (55,55); insert into shardddl1.tb1 values (56,56); insert into shardddl1.tb1 values (57,57); insert into shardddl1.tb1 values (58,58); insert into shardddl1.tb1 values (59,59); insert into shardddl1.tb1 values (60,60); commit; begin; insert into shardddl1.t_1 values (61,61); insert into shardddl1.t_1 values (62,62); insert into shardddl1.t_1 values (63,63); insert into shardddl1.t_1 values (64,64); insert into shardddl1.t_1 values (65,65); insert into shardddl1.t_1 values (66,66); insert into shardddl1.t_1 values (67,67); insert into shardddl1.t_1 values (68,68); insert into shardddl1.t_1 values (69,69); insert into shardddl1.t_1 values (70,70); commit; check diff failed 1-th time, retry later [Fri Apr 26 19:25:21 CST 2024] <<<<<< start DM-056 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:23 CST 2024] <<<<<< finish DM-056 pessimistic >>>>>> [Fri Apr 26 19:25:23 CST 2024] <<<<<< start DM-056 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:25:25 CST 2024] <<<<<< test case foreign_key success! >>>>>> start running case: [full_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:25:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 3-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully restart dm-worker1 rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:25:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" wait process worker1 exit... [Fri Apr 26 19:25:28 CST 2024] <<<<<< finish DM-056 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Fri Apr 26 19:25:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:25:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:25:29 CST 2024] <<<<<< start DM-057 pessimistic >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" rpc addr 127.0.0.1:8262 is alive restart dm-worker2 dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/full_mode/dm-task.yaml --remove-meta" wait process worker2 exit... wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:25:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:33 CST 2024] <<<<<< finish DM-057 pessimistic >>>>>> rpc addr 127.0.0.1:8263 is alive begin; insert into shardddl1.tb2 values (101,101); insert into shardddl1.tb2 values (102,102); insert into shardddl1.tb2 values (103,103); insert into shardddl1.tb2 values (104,104); insert into shardddl1.tb2 values (105,105); insert into shardddl1.tb2 values (106,106); insert into shardddl1.tb2 values (107,107); insert into shardddl1.tb2 values (108,108); insert into shardddl1.tb2 values (109,109); insert into shardddl1.tb2 values (110,110); commit; begin; insert into shardddl1.tb1 values (111,111); insert into shardddl1.tb1 values (112,112); insert into shardddl1.tb1 values (113,113); insert into shardddl1.tb1 values (114,114); insert into shardddl1.tb1 values (115,115); insert into shardddl1.tb1 values (116,116); insert into shardddl1.tb1 values (117,117); insert into shardddl1.tb1 values (118,118); insert into shardddl1.tb1 values (119,119); insert into shardddl1.tb1 values (120,120); commit; begin; insert into shardddl1.tb2 values (121,121); insert into shardddl1.tb2 values (122,122); insert into shardddl1.tb2 values (123,123); insert into shardddl1.tb2 values (124,124); insert into shardddl1.tb2 values (125,125); insert into shardddl1.tb2 values (126,126); insert into shardddl1.tb2 values (127,127); insert into shardddl1.tb2 values (128,128); insert into shardddl1.tb2 values (129,129); insert into shardddl1.tb2 values (130,130); commit; begin; insert into shardddl1.t_1 values (131,131); insert into shardddl1.t_1 values (132,132); insert into shardddl1.t_1 values (133,133); insert into shardddl1.t_1 values (134,134); insert into shardddl1.t_1 values (135,135); insert into shardddl1.t_1 values (136,136); insert into shardddl1.t_1 values (137,137); insert into shardddl1.t_1 values (138,138); insert into shardddl1.t_1 values (139,139); insert into shardddl1.t_1 values (140,140); commit; check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive begin; insert into shardddl1.tb2 values (201,201); insert into shardddl1.tb2 values (202,202); insert into shardddl1.tb2 values (203,203); insert into shardddl1.tb2 values (204,204); insert into shardddl1.tb2 values (205,205); insert into shardddl1.tb2 values (206,206); insert into shardddl1.tb2 values (207,207); insert into shardddl1.tb2 values (208,208); insert into shardddl1.tb2 values (209,209); insert into shardddl1.tb2 values (210,210); commit; begin; insert into shardddl1.tb1 values (211,211); insert into shardddl1.tb1 values (212,212); insert into shardddl1.tb1 values (213,213); insert into shardddl1.tb1 values (214,214); insert into shardddl1.tb1 values (215,215); insert into shardddl1.tb1 values (216,216); insert into shardddl1.tb1 values (217,217); insert into shardddl1.tb1 values (218,218); insert into shardddl1.tb1 values (219,219); insert into shardddl1.tb1 values (220,220); commit; begin; insert into shardddl1.tb2 values (221,221); insert into shardddl1.tb2 values (222,222); insert into shardddl1.tb2 values (223,223); insert into shardddl1.tb2 values (224,224); insert into shardddl1.tb2 values (225,225); insert into shardddl1.tb2 values (226,226); insert into shardddl1.tb2 values (227,227); insert into shardddl1.tb2 values (228,228); insert into shardddl1.tb2 values (229,229); insert into shardddl1.tb2 values (230,230); commit; begin; insert into shardddl1.t_1 values (231,231); insert into shardddl1.t_1 values (232,232); insert into shardddl1.t_1 values (233,233); insert into shardddl1.t_1 values (234,234); insert into shardddl1.t_1 values (235,235); insert into shardddl1.t_1 values (236,236); insert into shardddl1.t_1 values (237,237); insert into shardddl1.t_1 values (238,238); insert into shardddl1.t_1 values (239,239); insert into shardddl1.t_1 values (240,240); commit; check diff failed 1-th time, retry later [Fri Apr 26 19:25:34 CST 2024] <<<<<< start DM-058 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:36 CST 2024] <<<<<< finish DM-058 pessimistic >>>>>> [Fri Apr 26 19:25:36 CST 2024] <<<<<< start DM-058 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:36 CST 2024] <<<<<< finish DM-RESYNC_TXN_INTERRUPT optimistic >>>>>> [Fri Apr 26 19:25:36 CST 2024] <<<<<< start DM-STRICT_OPTIMISTIC_SINGLE_SOURCE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/single-source-strict-optimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:38 CST 2024] <<<<<< finish DM-STRICT_OPTIMISTIC_SINGLE_SOURCE optimistic >>>>>> [Fri Apr 26 19:25:38 CST 2024] <<<<<< start DM-STRICT_OPTIMISTIC_DOUBLE_SOURCE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-strict-optimistic.yaml --remove-meta" [Fri Apr 26 19:25:38 CST 2024] <<<<<< finish DM-058 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:25:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:25:39 CST 2024] <<<<<< start DM-059 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:25:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:25:40 CST 2024] <<<<<< finish DM-STRICT_OPTIMISTIC_DOUBLE_SOURCE optimistic >>>>>> [Fri Apr 26 19:25:40 CST 2024] <<<<<< start DM-131 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:25:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" check diff successfully dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:43 CST 2024] <<<<<< finish DM-131 optimistic >>>>>> dmctl test cmd: "start-task /tmp/dm_test/full_mode/dm-task.yaml --remove-meta" check diff failed 1-th time, retry later [Fri Apr 26 19:25:44 CST 2024] <<<<<< start DM-132 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:45 CST 2024] <<<<<< finish DM-059 pessimistic >>>>>> [Fri Apr 26 19:25:45 CST 2024] <<<<<< start DM-059 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/full_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/full_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:48 CST 2024] <<<<<< finish DM-132 pessimistic >>>>>> [Fri Apr 26 19:25:48 CST 2024] <<<<<< start DM-132 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:50 CST 2024] <<<<<< finish DM-132 optimistic >>>>>> check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... [Fri Apr 26 19:25:51 CST 2024] <<<<<< start DM-133 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:51 CST 2024] <<<<<< finish DM-059 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test" [Fri Apr 26 19:25:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 check diff failed 1-th time, retry later [Fri Apr 26 19:25:52 CST 2024] <<<<<< start DM-062 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:25:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Fri Apr 26 19:25:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:55 CST 2024] <<<<<< finish DM-133 pessimistic >>>>>> [Fri Apr 26 19:25:55 CST 2024] <<<<<< start DM-133 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:57 CST 2024] <<<<<< finish DM-062 pessimistic >>>>>> [Fri Apr 26 19:25:57 CST 2024] <<<<<< start DM-062 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task.yaml --remove-meta" [Fri Apr 26 19:25:57 CST 2024] <<<<<< finish DM-133 optimistic >>>>>> dmctl test cmd: "query-status test" check diff successfully [Fri Apr 26 19:25:58 CST 2024] <<<<<< start DM-134 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:25:59 CST 2024] <<<<<< finish DM-062 optimistic >>>>>> dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Fri Apr 26 19:26:00 CST 2024] <<<<<< start DM-063 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:01 CST 2024] <<<<<< finish DM-063 pessimistic >>>>>> [Fri Apr 26 19:26:01 CST 2024] <<<<<< start DM-063 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:26:02 CST 2024] <<<<<< finish DM-134 pessimistic >>>>>> [Fri Apr 26 19:26:02 CST 2024] <<<<<< start DM-134 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:04 CST 2024] <<<<<< finish DM-134 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:26:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:26:05 CST 2024] <<<<<< start DM-135 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:26:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:07 CST 2024] <<<<<< finish DM-135 pessimistic >>>>>> [Fri Apr 26 19:26:07 CST 2024] <<<<<< start DM-135 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:07 CST 2024] <<<<<< finish DM-063 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:08 CST 2024] <<<<<< start DM-064 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" [Fri Apr 26 19:26:09 CST 2024] <<<<<< finish DM-135 optimistic >>>>>> [Fri Apr 26 19:26:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:26:10 CST 2024] <<<<<< start DM-136 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task-2.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:12 CST 2024] <<<<<< finish DM-064 pessimistic >>>>>> [Fri Apr 26 19:26:12 CST 2024] <<<<<< start DM-064 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:14 CST 2024] <<<<<< finish DM-136 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:26:14 CST 2024] <<<<<< finish DM-064 optimistic >>>>>> [Fri Apr 26 19:26:15 CST 2024] <<<<<< start DM-137 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... [Fri Apr 26 19:26:15 CST 2024] <<<<<< start DM-065 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:17 CST 2024] <<<<<< finish DM-137 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:26:17 CST 2024] <<<<<< finish DM-065 pessimistic >>>>>> [Fri Apr 26 19:26:17 CST 2024] <<<<<< start DM-065 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:26:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:26:18 CST 2024] <<<<<< start DM-138 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:26:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:26:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:26:21 CST 2024] <<<<<< finish DM-065 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:22 CST 2024] <<<<<< finish DM-138 pessimistic >>>>>> [Fri Apr 26 19:26:22 CST 2024] <<<<<< start DM-138 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Fri Apr 26 19:26:22 CST 2024] <<<<<< start DM-066 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 check diff failed 1-th time, retry later [Fri Apr 26 19:26:24 CST 2024] <<<<<< finish DM-138 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task.yaml --remove-meta" [Fri Apr 26 19:26:25 CST 2024] <<<<<< start DM-139 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/full_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/full_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:26 CST 2024] <<<<<< finish DM-066 pessimistic >>>>>> [Fri Apr 26 19:26:26 CST 2024] <<<<<< start DM-066 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:28 CST 2024] <<<<<< finish DM-066 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:29 CST 2024] <<<<<< finish DM-139 pessimistic >>>>>> [Fri Apr 26 19:26:29 CST 2024] <<<<<< start DM-139 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... [Fri Apr 26 19:26:29 CST 2024] <<<<<< start DM-067 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:30 CST 2024] <<<<<< finish DM-139 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:26:31 CST 2024] <<<<<< test case full_mode success! >>>>>> start running case: [gbk] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/run.sh... Verbose mode = false dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:26:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Fri Apr 26 19:26:31 CST 2024] <<<<<< finish DM-067 pessimistic >>>>>> [Fri Apr 26 19:26:31 CST 2024] <<<<<< start DM-067 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" [Fri Apr 26 19:26:31 CST 2024] <<<<<< start DM-142 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:33 CST 2024] <<<<<< finish DM-067 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:26:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:26:34 CST 2024] <<<<<< start DM-068 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:26:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gbk/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/gbk/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:35 CST 2024] <<<<<< finish DM-142 pessimistic >>>>>> use sync_diff_inspector to check increment data check diff successfully check diff successfully data checked after one worker was killed try to kill worker port 8263 wait process dm-worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit worker2 was killed dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 [Fri Apr 26 19:26:34 CST 2024] <<<<<< finish test_multi_task_reduce_and_restart_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... [Fri Apr 26 19:26:36 CST 2024] <<<<<< start DM-143 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" prepare data start task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully prepare incremental data check incremental phase check diff successfully ERROR 1146 (42S02) at line 1: Table 'gbk.ddl1' doesn't exist run tidb sql failed 1-th time, retry later wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:38 CST 2024] <<<<<< finish DM-068 pessimistic >>>>>> [Fri Apr 26 19:26:38 CST 2024] <<<<<< start DM-068 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:38 CST 2024] <<<<<< finish DM-143 pessimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:39 CST 2024] <<<<<< start DM-145 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" ERROR 1146 (42S02) at line 1: Table 'gbk.ddl2_copy' doesn't exist run tidb sql failed 1-th time, retry later wait process dm-master.test exit... [Fri Apr 26 19:26:40 CST 2024] <<<<<< finish DM-068 optimistic >>>>>> dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... ERROR 1146 (42S02) at line 1: Table 'gbk.ddl3' doesn't exist run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:26:43 CST 2024] <<<<<< test case ha_cases2 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:43 CST 2024] <<<<<< finish DM-145 pessimistic >>>>>> [Fri Apr 26 19:26:43 CST 2024] <<<<<< start DM-145 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-worker.test exit... ERROR 1049 (42000) at line 1: Unknown database 'gbk3' run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:26:46 CST 2024] <<<<<< test case shardddl2_1 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } ERROR 1146 (42S02) at line 1: Table 'gbk.ddl4' doesn't exist run tidb sql failed 1-th time, retry later [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:47 CST 2024] <<<<<< finish DM-145 optimistic >>>>>> prepare data for invalid connection test check log contain failed 1-th time, retry later [Fri Apr 26 19:26:48 CST 2024] <<<<<< start DM-146 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Fri Apr 26 19:26:50 CST 2024] <<<<<< finish DM-146 pessimistic >>>>>> [Fri Apr 26 19:26:50 CST 2024] <<<<<< start DM-146 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:26:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:26:52 CST 2024] <<<<<< finish DM-146 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:26:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:26:53 CST 2024] <<<<<< start DM-147 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start test invalid connection with status running check count check diff successfully check test invalid connection with status running successfully dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl4_1/schema.sql -s mysql-replica-01" dmctl test cmd: "binlog replace test "alter table shardddl1.tb1 drop column b"" got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:26:56 CST 2024] <<<<<< finish DM-147 optimistic >>>>>> wait process dm-worker.test exit... [Fri Apr 26 19:26:57 CST 2024] <<<<<< start DM-148 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:26:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:26:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive start test invalid connection with status queueing check count check diff successfully check test invalid connection with status queueing successfully check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:27:01 CST 2024] <<<<<< finish DM-148 pessimistic >>>>>> [Fri Apr 26 19:27:01 CST 2024] <<<<<< start DM-148 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:27:03 CST 2024] <<<<<< finish DM-148 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:27:04 CST 2024] <<<<<< start DM-149 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start test invalid connection with status none check count check diff successfully check test invalid connection with status none successfully wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:27:08 CST 2024] <<<<<< finish DM-149 pessimistic >>>>>> [Fri Apr 26 19:27:08 CST 2024] <<<<<< start DM-149 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive start test inserting data after invalid connection check count dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check test inserting data after invalid connection successfully check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:27:12 CST 2024] <<<<<< finish DM-149 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:27:13 CST 2024] <<<<<< start DM-150 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start test adding UNIQUE on column with duplicate data check cancelled error dmctl test cmd: "query-status gbk" got=0 expected=1 command: query-status gbk origin SQL: \[ALTER TABLE gbk.invalid_conn_test1 ADD UNIQUE(i)\]: DDL ALTER TABLE `gbk`.`invalid_conn_test1` ADD UNIQUE(`i`) executed in background and met error count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "show-ddl-locks" got=1 expected=1 check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "resume-task gbk" got=3 expected=3 check test adding UNIQUE on column with duplicate data successfully check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:27:17 CST 2024] <<<<<< finish DM-150 pessimistic >>>>>> [Fri Apr 26 19:27:17 CST 2024] <<<<<< start DM-150 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> [Fri Apr 26 19:27:19 CST 2024] <<<<<< finish DM-150 optimistic >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:27:20 CST 2024] <<<<<< start DM-151 pessimistic >>>>>> rpc addr 127.0.0.1:8263 is alive start test invalid connection with status running (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "show-ddl-locks" got=1 expected=1 check count 2 check diff successfully check test invalid connection with status running (multi-schema change) successfully wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:27:25 CST 2024] <<<<<< finish DM-151 pessimistic >>>>>> [Fri Apr 26 19:27:25 CST 2024] <<<<<< start DM-151 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive start test invalid connection with status queueing (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 check count 2 check diff successfully check test invalid connection with status queueing (multi-schema change) successfully wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:27:32 CST 2024] <<<<<< finish DM-151 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive start test invalid connection with status none (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later [Fri Apr 26 19:27:33 CST 2024] <<<<<< start DM-152 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check count 2 check diff successfully check test invalid connection with status none (multi-schema change) successfully check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 2-th time, retry later rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive start test inserting data after invalid connection (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later check diff successfully check diff failed 1-th time, retry later check diff successfully check count 2 check diff successfully check test inserting data after invalid connection (multi-schema change) successfully check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:27:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:27:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive start test adding UNIQUE on column with duplicate data (multi-schema change) check cancelled error dmctl test cmd: "query-status gbk" got=0 expected=1 command: query-status gbk origin SQL: \[ALTER TABLE gbk.invalid_conn_test1 ADD UNIQUE(k), ADD UNIQUE(m)\]: DDL ALTER TABLE `gbk`.`invalid_conn_test1` ADD UNIQUE(`k`) executed in background and met error count: 0 != expected: 1, failed the 0-th time, will retry again check diff successfully check diff failed 1-th time, retry later got=1 expected=1 check test adding UNIQUE on column with duplicate data (multi-schema change) successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:27:48 CST 2024] <<<<<< finish DM-152 optimistic >>>>>> wait process dm-master.test exit... [Fri Apr 26 19:27:49 CST 2024] <<<<<< start DM-153 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 -s mysql-replica-01 --from-target" dmctl test cmd: "binlog replace test "alter table shardddl1.tb1 drop column b"" got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Fri Apr 26 19:27:52 CST 2024] <<<<<< finish DM-153 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:27:53 CST 2024] <<<<<< test case gbk success! >>>>>> start running case: [gtid] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/run.sh... Verbose mode = false [Fri Apr 26 19:27:53 CST 2024] <<<<<< start DM-154 optimistic >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:27:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:27:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 -s mysql-replica-01 --from-source" dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source1.yaml" [Fri Apr 26 19:27:55 CST 2024] <<<<<< finish DM-154 optimistic >>>>>> [Fri Apr 26 19:27:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Fri Apr 26 19:27:56 CST 2024] <<<<<< start DM-155 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source2.yaml" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml --remove-meta" got=2 expected=2 restart master restart dm-master check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master exit... wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... [Fri Apr 26 19:28:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:28:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time restart worker1 restart dm-worker1 rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:28:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker1 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source1.yaml" [Fri Apr 26 19:28:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker1 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source2.yaml" wait process worker1 exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml --remove-meta" wait process worker1 exit... wait process worker1 exit... check diff successfully new_gtid1 fe276a06-03bd-11ef-b264-0601375cf6a9:6 new_gtid2 feab8bba-03bd-11ef-9071-0601375cf6a9:6 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:28:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:28:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker1 exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml" wait process worker1 exit... check diff successfully dmctl test cmd: "stop-task test" wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:28:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Fri Apr 26 19:28:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> [Fri Apr 26 19:28:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml" restart worker1 restart dm-worker1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process worker1 exit... wait process dm-master.test exit... wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:28:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive wait process dm-worker.test exit... restart worker2 restart dm-worker2 wait process dm-worker.test exit... wait process worker2 exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:28:24 CST 2024] <<<<<< test case gtid success! >>>>>> start running case: [ha_cases] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:28:24 CST 2024] <<<<<< start test_exclusive_relay >>>>>> start DM worker and master cluster [Fri Apr 26 19:28:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-standalone.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:28:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8263 is alive rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:28:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time restart master restart dm-master rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" [Fri Apr 26 19:28:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master exit... wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1 worker2" got=3 expected=3 dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" dmctl test cmd: "list-member --worker" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source show -s mysql-replica-02" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Fri Apr 26 19:28:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table restart worker2 restart dm-worker2 wait process worker2 exit... wait process worker2 exit... [Fri Apr 26 19:28:37 CST 2024] <<<<<< finish test_exclusive_relay >>>>>> [Fri Apr 26 19:28:37 CST 2024] <<<<<< start test_exclusive_relay_2 >>>>>> start DM worker and master cluster [Fri Apr 26 19:28:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-standalone.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process worker2 exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process worker2 exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:28:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" [Fri Apr 26 19:28:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" wait process worker2 exit... wait process worker2 exit... dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process worker2 exit... got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:28:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 [Fri Apr 26 19:28:45 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8263 is alive rpc addr 127.0.0.1:8264 is alive kill dm-worker1 restart worker2 restart dm-worker2 wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --name worker3" wait process worker2 exit... got=1 expected=1 [Fri Apr 26 19:28:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:28:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member --name worker3" got=1 expected=1 dmctl test cmd: "list-member --name worker1" got=1 expected=1 kill dm-worker2 rpc addr 127.0.0.1:8263 is alive wait process dm-worker2 exit... restart master restart dm-master wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "operate-source show -s mysql-replica-02" got=1 expected=1 [Fri Apr 26 19:28:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker2" got=1 expected=1 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master exit... process dm-master already exit wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:28:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later [Fri Apr 26 19:29:00 CST 2024] <<<<<< finish test_exclusive_relay_2 >>>>>> [Fri Apr 26 19:29:00 CST 2024] <<<<<< start test_last_bound >>>>>> [Fri Apr 26 19:29:00 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table check log contain failed 1-th time, retry later check log contain failed 1-th time, retry later import prepare data start DM worker and master cluster [Fri Apr 26 19:29:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:29:02 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:29:02 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time restart worker1 restart dm-worker1 wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process worker1 exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Fri Apr 26 19:29:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker1 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" wait process worker1 exit... [Fri Apr 26 19:29:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker1 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:29:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive check log contain failed 1-th time, retry later start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check log contain failed 1-th time, retry later check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase check log contain failed 1-th time, retry later use sync_diff_inspector to check increment data check diff successfully [Fri Apr 26 19:29:16 CST 2024] <<<<<< finish test_running >>>>>> worker1bound "mysql-replica-01" worker2bound "mysql-replica-02" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" restart worker1 restart dm-worker1 got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:29:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 kill dm-worker1 rpc addr 127.0.0.1:8262 is alive check log contain failed 1-th time, retry later wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process dm-worker2 exit... check log contain failed 1-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker1 [Fri Apr 26 19:29:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive start worker2 [Fri Apr 26 19:29:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker1 --name worker2" got=1 expected=1 got=1 expected=1 kill dm-worker1 restart worker1 restart dm-worker1 wait process worker1 exit... wait process dm-worker1 exit... wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:29:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process dm-worker2 exit... rpc addr 127.0.0.1:8262 is alive wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" restart master restart dm-master got=2 expected=2 start worker2 [Fri Apr 26 19:29:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master exit... rpc addr 127.0.0.1:8263 is alive start worker1 [Fri Apr 26 19:29:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member --name worker2 --name worker1" got=1 expected=1 got=1 expected=1 kill dm-worker1 wait process dm-worker1 exit... [Fri Apr 26 19:29:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process dm-worker2 exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker3 [Fri Apr 26 19:29:37 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8264 is alive start worker4 [Fri Apr 26 19:29:38 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "list-member --name worker3 --name worker4" got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-relay -s mysql-replica-01 worker3" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker4" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 check diff successfully kill dm-worker3 wait process dm-worker3 exit... wait process dm-worker3 exit... process dm-worker3 already exit kill dm-worker4 wait process dm-worker4 exit... wait process dm-worker4 exit... process dm-worker4 already exit dmctl test cmd: "list-member --name worker3 --name worker4" got=2 expected=2 start worker1 [Fri Apr 26 19:29:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive start worker2 [Fri Apr 26 19:29:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker1 --name worker2" got=1 expected=1 got=1 expected=1 num1 1 num2 2 [Fri Apr 26 19:29:50 CST 2024] <<<<<< finish test_last_bound >>>>>> [Fri Apr 26 19:29:50 CST 2024] <<<<<< start test_config_name >>>>>> [Fri Apr 26 19:29:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join1.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:29:50 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/ha_cases/dm-master-join2.toml >>>>>> check log contain failed 1-th time (file not exist), retry later [Fri Apr 26 19:29:52 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/ha_cases/dm-master-join2.toml >>>>>> rpc addr 127.0.0.1:8361 is alive [Fri Apr 26 19:29:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:29:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/ha_cases/dm-worker2.toml >>>>>> [Fri Apr 26 19:29:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/ha_cases/dm-worker2.toml >>>>>> rpc addr 127.0.0.1:8263 is alive [Fri Apr 26 19:29:54 CST 2024] <<<<<< finish test_config_name >>>>>> [Fri Apr 26 19:29:54 CST 2024] <<<<<< start test_join_masters_and_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:29:53 CST 2024] <<<<<< finish DM-155 optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:29:59 CST 2024] <<<<<< test case shardddl4_1 success! >>>>>> start running case: [sharding] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:29:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:30:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding/source1.yaml" [Fri Apr 26 19:30:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding/source2.yaml" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "query-status test" got=2 expected=2 check sync diff for full dump and load check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" [Fri Apr 26 19:30:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 dmctl test cmd: "resume-task test" check sync diff for the first increment replication check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive query-status from unique master dmctl test cmd: "query-status" got=1 expected=1 [Fri Apr 26 19:30:07 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time check diff successfully check sync diff for the second increment replication check diff successfully check sync diff for the third increment replication check diff successfully checksum before drop/truncate: checksum: 2273109362, checksum after drop/truncate: checksum: 2273109362 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:30:15 CST 2024] <<<<<< test case sharding success! >>>>>> start running case: [sequence_sharding] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:30:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:30:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:30:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8361 alive the 2-th time rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-task.yaml " check diff successfully [Fri Apr 26 19:30:22 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time rpc addr 127.0.0.1:8461 is alive check diff successfully dmctl test cmd: "query-status sequence_sharding" got=0 expected=2 command: query-status sequence_sharding detect inconsistent DDL sequence count: 0 != expected: 2, failed the 0-th time, will retry again got=2 expected=2 [Fri Apr 26 19:30:29 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time rpc addr 127.0.0.1:8561 is alive [Fri Apr 26 19:30:35 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status" got=1 expected=1 dmctl test cmd: "query-status" got=1 expected=1 dmctl test cmd: "query-status" got=1 expected=1 dmctl test cmd: "query-status" got=1 expected=1 join worker with dm-master1 endpoint [Fri Apr 26 19:30:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker-join2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker --name=worker2" got=1 expected=1 kill dm-master-join1 dmctl test cmd: "resume-task sequence_sharding" dmctl test cmd: "query-status sequence_sharding" got=2 expected=2 dmctl test cmd: "stop-task sequence_sharding" dmctl test cmd: "start-task /tmp/dm_test/sequence_sharding/task.yaml" wait process dm-master-join1 exit... dmctl test cmd: "query-status sequence_sharding" got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master-join1 exit... wait process dm-master.test exit... wait process dm-master-join1 exit... process dm-master-join1 already exit dmctl test cmd: "list-member --worker --name=worker2" got=1 expected=1 join worker with 5 masters endpoint [Fri Apr 26 19:30:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8262 alive the 2-th time wait process dm-worker.test exit... wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8262 alive the 3-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:30:44 CST 2024] <<<<<< test case sequence_sharding success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-272/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } wait for rpc addr 127.0.0.1:8262 alive the 4-th time [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } rpc addr 127.0.0.1:8262 is alive query-status from master2 dmctl test cmd: "query-status" got=1 expected=1 [Fri Apr 26 19:30:46 CST 2024] <<<<<< finish test_join_masters_and_worker >>>>>> [Fri Apr 26 19:30:46 CST 2024] <<<<<< start test_standalone_running >>>>>> 4 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master standalone cluster [Fri Apr 26 19:30:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master1.toml >>>>>> [Fri Apr 26 19:30:58 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master2.toml >>>>>> [Fri Apr 26 19:30:58 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Fri Apr 26 19:31:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task.yaml " use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task2.yaml" [Fri Apr 26 19:31:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task2.yaml" dmctl test cmd: "query-status" got=2 expected=2 kill worker2 wait process dm-worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test2" got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task2.yaml" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 [Fri Apr 26 19:31:13 CST 2024] <<<<<< finish test_standalone_running >>>>>> 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:31:22 CST 2024] <<<<<< test case ha_cases success! >>>>>> start running case: [http_proxies] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit test dm grpc proxy env setting checking for http_proxy=http://127.0.0.1:8080 [Fri Apr 26 19:31:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected [Fri Apr 26 19:31:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit test dm grpc proxy env setting checking for https_proxy=https://127.0.0.1:8080 [Fri Apr 26 19:31:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected [Fri Apr 26 19:31:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit test dm grpc proxy env setting checking for no_proxy=localhost,127.0.0.1 [Fri Apr 26 19:31:42 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected [Fri Apr 26 19:31:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:31:53 CST 2024] <<<<<< test case http_proxies success! >>>>>> start running case: [lightning_load_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM master, workers and sources [Fri Apr 26 19:31:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:31:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_load_task/source1.yaml" [Fri Apr 26 19:31:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_load_task/source2.yaml" [Fri Apr 26 19:31:58 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8264 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task2.yaml --remove-meta" dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 got=1 expected=1 test worker restart wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 [Fri Apr 26 19:32:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 got=1 expected=1 test_transfer_two_sources wait process dm-worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task3.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status load_task3" got=1 expected=1 [Fri Apr 26 19:32:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task4.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status load_task4" got=1 expected=1 [Fri Apr 26 19:32:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 wait process dm-worker3 exit... wait process dm-worker3 exit... process dm-worker3 already exit dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 2-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 3-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 4-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status" got=3 expected=3 got=1 expected=1 [Fri Apr 26 19:32:28 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status" got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 0-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 1-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 2-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 3-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 4-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 5-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 6-th time, will retry again got=4 expected=4 check diff successfully check diff successfully check diff successfully check diff successfully 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:32:50 CST 2024] <<<<<< test case lightning_load_task success! >>>>>> start running case: [lightning_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit Starting PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:2379; Connection refused 2024-04-26 19:32:50.878250 W | pkg/fileutil: check file permission: directory "/tmp/dm_test/lightning_mode/pd" exist, but the permission is "drwxr-xr-x". The recommended permission is "-rwx------" to prevent possible unprivileged access to the data. % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 73337 0 --:--:-- --:--:-- --:--:-- 104k % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 93871 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 117 100 117 0 0 180k 0 --:--:-- --:--:-- --:--:-- 114k {"connections":0,"version":"8.0.11-TiDB-v7.5.1-43-ge406d5780b","git_hash":"e406d5780b18a1f2aaf6230cde4b6403991e228d"}[Fri Apr 26 19:33:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:33:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_mode/source1.yaml" [Fri Apr 26 19:33:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_mode/source2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-task.yaml" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Fri Apr 26 19:33:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-task-dup.yaml --remove-meta" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "stage": "Paused" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "stage": "Paused" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "stage": "Paused" count: 0 != expected: 1, failed the 2-th time, will retry again got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=0 expected=2 command: query-status test unit": "Sync" count: 0 != expected: 2, failed the 0-th time, will retry again got=0 expected=2 command: query-status test unit": "Sync" count: 0 != expected: 2, failed the 1-th time, will retry again got=1 expected=2 command: query-status test unit": "Sync" count: 1 != expected: 2, failed the 2-th time, will retry again got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/lightning_mode/dm-task-dup.yaml --remove-meta" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test checksum mismatched, KV number in source files: 6, KV number in TiDB cluster: 3 count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test checksum mismatched, KV number in source files: 6, KV number in TiDB cluster: 3 count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test checksum mismatched, KV number in source files: 6, KV number in TiDB cluster: 3 count: 0 != expected: 1, failed the 2-th time, will retry again got=1 expected=1 got=2 expected=2 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=0 expected=2 command: query-status test unit": "Sync" count: 0 != expected: 2, failed the 0-th time, will retry again got=0 expected=2 command: query-status test unit": "Sync" count: 0 != expected: 2, failed the 1-th time, will retry again got=1 expected=2 command: query-status test unit": "Sync" count: 1 != expected: 2, failed the 2-th time, will retry again got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-task.yaml --remove-meta" check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff successfully check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/lightning_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/lightning_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files check no password in log Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:34:09 CST 2024] <<<<<< test case lightning_mode success! >>>>>> start running case: [metrics] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check dashboard data source check dashboard data source success [Fri Apr 26 19:34:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:34:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/metrics/source1.yaml" [Fri Apr 26 19:34:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/metrics/source2.yaml" dmctl test cmd: "start-task /tmp/dm_test/metrics/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=3 expected=3 wait for valid metric of dm_syncer_binlog_file{node="syncer",source_id="mysql-replica-01",task="test"} for 1-th time, got value: '1' wait for valid metric of dm_syncer_binlog_file{node="master",source_id="mysql-replica-01",task="test"} for 1-th time, got value: '1' wait for valid metric of dm_syncer_replication_lag_sum{source_id="mysql-replica-01",task="test",worker="worker1"} for 1-th time, got value: '0' check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check ddl lag done! wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:34:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:34:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully check dml/skip lag done! wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:34:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:34:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check zero job done! wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Fri Apr 26 19:34:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive make a dml job check diff successfully dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:34:45 CST 2024] <<<<<< test case metrics success! >>>>>> start running case: [new_collation_off] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. [Fri Apr 26 19:34:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:34:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-task.yaml" check data check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:34:59 CST 2024] <<<<<< test case new_collation_off success! >>>>>> start running case: [only_dml] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:34:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:35:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/only_dml/source1.yaml" [Fri Apr 26 19:35:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/only_dml/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-task.yaml " check diff successfully PID of insert_data is 54880 dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000002 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000003 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000004 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000005 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000006 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000007 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000008 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000009 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 9-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000010 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000012 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000013 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000014 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000015 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000016 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000017 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000018 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000019 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 9-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000020 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000021 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000023 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000024 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000025 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000026 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000027 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000028 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000029 -s mysql-replica-01" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 9-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000030 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000031 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000032 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000034 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000035 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000036 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000037 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000038 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000039 -s mysql-replica-02" earliest_relay_log1: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 earliest_relay_log2: dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001 purge relay log failed 9-th time, retry later check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh: line 44: 54880 Terminated insert_data 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:35:51 CST 2024] <<<<<< test case only_dml success! >>>>>> start running case: [openapi] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:35:51 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:35:52 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time wait for rpc addr 127.0.0.1:8361 alive the 2-th time rpc addr 127.0.0.1:8361 is alive [Fri Apr 26 19:36:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:36:04 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: RELAY ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) dmctl test cmd: "query-status -s mysql-01" got=2 expected=2 got=1 expected=1 ('enable_relay_failed resp=', {u'error_code': 46004, u'error_msg': u'[code=46004:class=scheduler:scope=internal:level=medium], Message: dm-worker with name [no-worker] not exists'}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 ('get_source_status_failed resp=', {u'error_code': 46008, u'error_msg': u'[code=46008:class=scheduler:scope=internal:level=medium], Message: source config with ID no-mysql not exists'}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 1, u'data': [{u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('disable_relay_failed resp=', {u'error_code': 46004, u'error_msg': u'[code=46004:class=scheduler:scope=internal:level=medium], Message: dm-worker with name [no-worker] not exists'}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 ('get_source_status_success_no_relay resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('list_source_with_status_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'task_name_list': [], u'flavor': u'mysql', u'password': u'******', u'port': 3306, u'status_list': [{u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'', u'relay_catch_up_master': False, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}]}) ('get_source_status_success resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'fe276a06-03bd-11ef-b264-0601375cf6a9.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 488)', u'relay_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 delete_source_success >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: RELAY SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: SOURCE ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source_failed resp=', {u'error_code': 10001, u'error_msg': u'[code=10001:class=database:scope=upstream:level=high], Message: database driver error, RawCause: dial tcp :0: connect: connection refused, Workaround: Please check the database connection and the database config in configuration file.'}) ('update_source1_without_password_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': None, u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('list_source_with_status_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'task_name_list': [], u'flavor': u'mysql', u'password': u'******', u'port': 3306, u'status_list': [{u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}]}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 ('get_source_schemas_and_tables_success schema_resp=', [u'information_schema', u'Upper_DB', u'Upper_DB1', u'Upper_Db_IGNORE', u'adjust_gtid', u'async_checkpoint_flush', u'binlog_parse', u'check-task', u'checkpoint_transaction', u'checktask', u'checktask1', u'dm_syncer', u'dm_syncer_do_db', u'dm_syncer_ignore_db', u'downstream_diff_index1', u'downstream_more_column1', u'drop_column_with_index', u'dup_event', u'expr_filter', u'extend_column1', u'fake_rotate_event', u'foreign_key', u'full/mode', u'full_mode', u'gbk', u'gbk3', u'gtid', u'ha_test', u'ignore_db', u'lightning_mode', u'load_task1', u'load_task2', u'load_task4', u'metrics', u'mysql', u'only_dml', u'openapi', u'performance_schema', u'sys']) ('get_source_schemas_and_tables_success table_resp=', [u't1']) ('get_source_schemas_and_tables_success schema_resp=', [u'information_schema', u'Upper_DB', u'Upper_DB1', u'Upper_Db_IGNORE', u'adjust_gtid', u'async_checkpoint_flush', u'binlog_parse', u'check-task', u'checkpoint_transaction', u'checktask', u'checktask1', u'db-name', u'dm_syncer', u'dm_syncer_do_db', u'dm_syncer_ignore_db', u'downstream_diff_index1', u'downstream_more_column1', u'drop_column_with_index', u'dup_event', u'expr_filter', u'extend_column1', u'fake_rotate_event', u'foreign_key', u'full/mode', u'full_mode', u'gbk', u'gbk3', u'gtid', u'ha_test', u'ignore_db', u'lightning_mode', u'load_task1', u'load_task2', u'load_task4', u'metrics', u'mysql', u'only_dml', u'openapi', u'performance_schema', u'sys']) ('get_source_schemas_and_tables_success table_resp=', [u't1']) delete_source_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('delete_source_failed msg=', {u'error_code': 46008, u'error_msg': u'[code=46008:class=scheduler:scope=internal:level=medium], Message: source config with ID mysql-01 not exists'}) ('list_source_with_reverse resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: SOURCE SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: SHARD TASK ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-02'}]}) ('create_task_failed resp=', {u'error': u'error in openapi3filter.RequestError: request body has an error: doesn\'t match the schema: Error at "/task/shard_mode": value is not one of the allowed values'}) ('create_shard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'binlog_filter_rule': {u'rule-2': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}, u'rule-1': {u'ignore_event': [u'delete']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'rule-1'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'rule-2'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-shard" got=2 expected=2 dmctl test cmd: "query-status test-shard" got=2 expected=2 check diff successfully ('get_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name not a task name not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_illegal_char_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name t-\xcb!s`t not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 6, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3103)', u'syncer_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-12', u'synced': False, u'total_tps': 0, u'unresolved_groups': [{u'synced': None, u'first_location': u'', u'unsynced': [u"this DM-worker doesn't receive any shard DDL of this group"], u'target': u'`openapi`.`t`', u'ddl_list': None}], u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 2344)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': [u'ALTER TABLE `openapi`.`t` ADD COLUMN `aaa` INT'], u'recent_tps': 0, u'total_events': 6, u'master_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-11', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 2511)', u'syncer_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-10', u'synced': False, u'total_tps': 0, u'unresolved_groups': [{u'synced': [u'`openapi`.`t`'], u'first_location': u'position: (dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 2384), gtid-set: feab8bba-03bd-11ef-9071-0601375cf6a9:1-10', u'unsynced': None, u'target': u'`openapi`.`t`', u'ddl_list': [u'ALTER TABLE `openapi`.`t` ADD COLUMN `aaa` INT']}], u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 2307)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'binlog_filter_rule': {u'mysql-01-filter-rule-0': {u'ignore_event': [u'delete']}, u'mysql-02-filter-rule-0': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-02-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-01-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-02'}, {u'source_name': u'mysql-01'}]}}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'binlog_filter_rule': {u'mysql-01-filter-rule-0': {u'ignore_event': [u'delete']}, u'mysql-02-filter-rule-0': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-01-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-02-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}]}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: SHARD TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: MULTI TASK ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}}) dmctl test cmd: "query-status test-1" got=2 expected=2 dmctl test cmd: "query-status test-1" got=2 expected=2 ('get_task_list_with_status resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-02'}, {u'source_name': u'mysql-01'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3891)', u'syncer_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3891)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3395)', u'syncer_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3395)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-1'}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-2'}}) dmctl test cmd: "query-status test-2" got=2 expected=2 dmctl test cmd: "query-status test-2" got=1 expected=2 command: query-status test-2 "stage": "Running" count: 1 != expected: 2, failed the 0-th time, will retry again got=2 expected=2 ('get_task_list_with_status resp=', {u'total': 2, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3891)', u'syncer_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3891)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3395)', u'syncer_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3395)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-1'}, {u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-2', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3891)', u'syncer_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3891)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-2', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3395)', u'syncer_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 3395)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-2'}]}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: MULTI TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: NO SHARD TASK ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-02'}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}}) dmctl test cmd: "query-status test-no-shard" got=2 expected=2 dmctl test cmd: "query-status test-no-shard" got=2 expected=2 check diff successfully ('get_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name not a task name not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-23', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 4679)', u'syncer_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-22', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 4425)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 4279)', u'syncer_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-18', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 4005)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}]}) delete_source_with_force_success ('get_task_status_success resp=', {u'total': 1, u'data': [{u'source_name': u'mysql-02', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 4279)', u'syncer_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-18', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 4005)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}]}) ('get_task_schema_success schema resp=', [u'openapi']) ('get_task_schema_success table resp=', [u't2']) ('get_task_schema_success create table resp=', {u'table_name': u't2', u'schema_create_sql': u'CREATE TABLE `t2` ( `i` tinyint(4) DEFAULT NULL, `j` int(11) DEFAULT NULL, UNIQUE KEY `j` (`j`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin', u'schema_name': u'openapi'}) ('get_task_schema_success table resp=', [u't2']) ('delete_task_failed resp=', {u'error_code': 49001, u'error_msg': u'[code=49001:class=openapi:scope=internal:level=high], Message: task test-no-shard have running subtasks, please stop them or delete task with force.'}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: NO SHARD TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: TASK TEMPLATES ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_task_template_failed resp=', {u'error': u'error in openapi3filter.RequestError: request body has an error: doesn\'t match the schema: Error at "/shard_mode": value is not one of the allowed values'}) ('create_task_template_success resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}) ('list_task_template resp=', {u'total': 1, u'data': [{u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}]}) ('get_task_template resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}) ('update_task_template_success resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'full', u'name': u'test-1'}) delete_task_template ('list_task_template resp=', {u'total': 0, u'data': []}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}}) dmctl test cmd: "query-status test-1" got=2 expected=2 dmctl test cmd: "query-status test-1" got=2 expected=2 ('import_task_template resp=', {u'success_task_list': [u'test-1'], u'failed_task_list': []}) ('list_task_template resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}]}) dmctl test cmd: "config task test-1 --path /tmp/dm_test/openapi/get_task_from_task.yaml" dmctl test cmd: "config task-template test-1 --path /tmp/dm_test/openapi/get_task_from_task_template.yaml" delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: TASK TEMPLATES >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: NO SHARD TASK DUMP STATUS wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:36:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:36:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-02'}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard-dump-status'}}) dmctl test cmd: "query-status test-no-shard-dump-status" got=2 expected=2 dmctl test cmd: "query-status test-no-shard-dump-status" got=2 expected=2 got=2 expected=2 ('check_dump_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-no-shard-dump-status', u'unresolved_ddl_lock_id': u'', u'worker_name': u'worker2', u'dump_status': {u'estimate_total_rows': 0, u'finished_bytes': 0, u'completed_tables': 0, u'total_tables': 0, u'finished_rows': 0}, u'unit': u'Dump', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-no-shard-dump-status', u'unresolved_ddl_lock_id': u'', u'worker_name': u'worker1', u'dump_status': {u'estimate_total_rows': 0, u'finished_bytes': 0, u'completed_tables': 0, u'total_tables': 0, u'finished_rows': 0}, u'unit': u'Dump', u'stage': u'Running'}]}) wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit ('get_task_status_success_but_worker_meet_error resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-no-shard-dump-status', u'worker_name': u'', u'error_msg': u'[code=38029:class=dm-master:scope=internal:level=high], Message: mysql-01 relevant worker-client not found, Workaround: Please use list-member command to see if the some workers are offline.', u'unit': u'', u'stage': u''}, {u'source_name': u'mysql-02', u'name': u'test-no-shard-dump-status', u'worker_name': u'', u'error_msg': u'[code=38029:class=dm-master:scope=internal:level=high], Message: mysql-02 relevant worker-client not found, Workaround: Please use list-member command to see if the some workers are offline.', u'unit': u'', u'stage': u''}]}) Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-worker.test already exit [Fri Apr 26 19:36:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:36:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: NO SHARD TASK DUMP STATUS SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: COMPLEX OPERATION ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-complex'}}) dmctl test cmd: "query-status test-complex" got=2 expected=2 ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-complex'}]}) dmctl test cmd: "query-status test-complex" got=2 expected=2 check diff successfully ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-complex', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-33', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 6589)', u'syncer_binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-32', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 6335)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-complex', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-29', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 6441)', u'syncer_binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-28', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 6167)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}]}) check diff successfully delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: COMPLEX OPERATION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: TEST TASK WITH IGNORE CHECK ITEMS ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-02'}]}) ('create_task_with_precheck resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'name': u'test-no-ignore-no-error', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u''], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-no-ignore-no-error" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_task_with_precheck resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 21,\n\t\t\t\t"name": "sharding table `openapi`.`t` consistency checking",\n\t\t\t\t"desc": "check consistency of sharding table structures",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "sourceID mysql-01 table {openapi t} of sharding `openapi`.`t` have auto-increment key, please make sure them don\'t conflict in target table!"\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "sourceID mysql-02 table {openapi t} of sharding `openapi`.`t` have auto-increment key, please make sure them don\'t conflict in target table!"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "If happen conflict, please handle it by yourself. You can refer to https://docs.pingcap.com/tidb-data-migration/stable/shard-merge-best-practices/#handle-conflicts-between-primary-keys-or-unique-indexes-across-multiple-sharded-tables",\n\t\t\t\t"extra": "auto-increment key checking"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 22,\n\t\t\t"successful": 21,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'name': u'test-no-ignore-has-warn', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u''], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-no-ignore-has-warn" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_task_with_precheck resp=', {u'error_code': 26005, u'error_msg': u'[code=26005:class=dm-master:scope=internal:level=medium], Message: fail to check synchronization configuration with type: check was failed, please see detail\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 21,\n\t\t\t\t"name": "sharding table `openapi`.`t` consistency checking",\n\t\t\t\t"desc": "check consistency of sharding table structures",\n\t\t\t\t"state": "fail",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "column length mismatch (self: 2 vs other: 3)",\n\t\t\t\t\t\t"self": "sourceID mysql-02 table {openapi t} columns [i j]",\n\t\t\t\t\t\t"other": "sourceID mysql-01 table `openapi`.`t` columns [id i j]"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "please set same table structure for sharding tables",\n\t\t\t\t"extra": "error on sharding `openapi`.`t`"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": false,\n\t\t\t"total": 22,\n\t\t\t"successful": 21,\n\t\t\t"failed": 1,\n\t\t\t"warning": 0\n\t\t}\n\t}'}) dmctl test cmd: "query-status test-no-ignore-has-error" got=1 expected=1 ('create_task_with_precheck resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'name': u'test-has-ignore-without-error', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u'schema_of_shard_tables'], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-has-ignore-without-error" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: TEST TASK WITH IGNORE CHECK ITEMS SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: DELETE TASK WITH STOPPED DOWNSTREAM ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}}) dmctl test cmd: "query-status test-no-shard" got=2 expected=2 tidb_server_num tidb-server alive wait process tidb-server exit... process tidb-server already exit ('delete_task_failed resp=', {u'error_code': 10001, u'error_msg': u'[code=10001:class=database:scope=downstream:level=high], Message: database driver error, RawCause: dial tcp 127.0.0.1:4000: connect: connection refused, Workaround: Please check the database connection and the database config in configuration file.'}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: DELETE TASK WITH STOPPED DOWNSTREAM SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: START TASK WITH CONDITION ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-02'}]}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_no_source_meta'}}) dmctl test cmd: "query-status incremental_task_no_source_meta" got=2 expected=2 ('start_task_failed resp=', {u'error_code': 20022, u'error_msg': u'[code=20022:class=config:scope=internal:level=medium], Message: mysql-instance(mysql-02) must set meta for task-mode incremental, Workaround: Please check the `meta` config in task configuration file.'}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 10,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 20,\n\t\t\t"successful": 19,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01', u'binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-58', u'binlog_name': u'dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001', u'binlog_pos': 11460}, {u'source_name': u'mysql-02', u'binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-54', u'binlog_name': u'dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001', u'binlog_pos': 11906}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_gtid'}}) dmctl test cmd: "query-status incremental_task_use_gtid" got=2 expected=2 dmctl test cmd: "query-status incremental_task_use_gtid" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_start_time'}}) dmctl test cmd: "query-status incremental_task_use_start_time" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_use_start_time" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_start_time_after_create'}}) dmctl test cmd: "query-status incremental_task_use_start_time_after_create" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_use_start_time_after_create" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 10,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 20,\n\t\t\t"successful": 19,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01', u'binlog_gtid': u'fe276a06-03bd-11ef-b264-0601375cf6a9:1-70', u'binlog_name': u'dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001', u'binlog_pos': 13696}, {u'source_name': u'mysql-02', u'binlog_gtid': u'feab8bba-03bd-11ef-9071-0601375cf6a9:1-66', u'binlog_name': u'dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001', u'binlog_pos': 14442}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_both_gtid_start_time'}}) dmctl test cmd: "query-status incremental_task_both_gtid_start_time" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_both_gtid_start_time" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:37:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:37:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_no_duration_but_error'}}) dmctl test cmd: "query-status incremental_task_no_duration_but_error" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_no_duration_but_error" got=2 expected=2 start_task_with_condition success run tidb sql failed 1-th time, retry later delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: STOP TASK WITH CONDITION ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-02'}]}) wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:37:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:37:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test_wait_time_on_stop'}}) dmctl test cmd: "query-status test_wait_time_on_stop" got=2 expected=2 dmctl test cmd: "query-status test_wait_time_on_stop" got=2 expected=2 check diff successfully error check wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:37:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:37:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: REVERSE HTTPS 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:37:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/openapi/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:37:36 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/openapi/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time wait for rpc addr 127.0.0.1:8361 alive the 2-th time rpc addr 127.0.0.1:8361 is alive [Fri Apr 26 19:37:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/openapi/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:37:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/openapi/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('create_source_success_https resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_success_https resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('list_source_with_reverse_https resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:37:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:37:54 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time wait for rpc addr 127.0.0.1:8361 alive the 2-th time rpc addr 127.0.0.1:8361 is alive [Fri Apr 26 19:38:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:38:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: REVERSE HTTPS ('list_master_success resp=', {u'total': 2, u'data': [{u'name': u'master1', u'addr': u'http://127.0.0.1:8291', u'alive': True, u'leader': True}, {u'name': u'master2', u'addr': u'http://127.0.0.1:8292', u'alive': True, u'leader': False}]}) ('list_worker_success resp=', {u'total': 2, u'data': [{u'bound_source_name': u'', u'bound_stage': u'free', u'addr': u'127.0.0.1:8262', u'name': u'worker1'}, {u'bound_source_name': u'', u'bound_stage': u'free', u'addr': u'127.0.0.1:8263', u'name': u'worker2'}]}) delete_master_with_retry_success ('list_master_success resp=', {u'total': 1, u'data': [{u'name': u'master1', u'addr': u'http://127.0.0.1:8291', u'alive': True, u'leader': True}]}) ('delete_worker_failed resp=', {u'error_code': 46005, u'error_msg': u'[code=46005:class=scheduler:scope=internal:level=medium], Message: dm-worker with name worker1 is still online, Workaround: Please shut it down first.'}) wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit delete_worker_with_retry_success ('list_worker_success resp=', {u'total': 1, u'data': [{u'bound_source_name': u'', u'bound_stage': u'offline', u'addr': u'127.0.0.1:8263', u'name': u'worker2'}]}) 2 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:38:20 CST 2024] <<<<<< test case openapi success! >>>>>> start running case: [s3_dumpling_lightning] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit Starting PD... Release Version: v7.5.1-5-g584533652 Edition: Community Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b Git Branch: release-7.5 UTC Build Time: 2024-04-03 10:04:14 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:2379; Connection refused 2024-04-26 19:38:20.654631 W | pkg/fileutil: check file permission: directory "/tmp/dm_test/s3_dumpling_lightning.downstream/pd" exist, but the permission is "drwxr-xr-x". The recommended permission is "-rwx------" to prevent possible unprivileged access to the data. % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 10 100 10 0 0 17211 0 --:--:-- --:--:-- --:--:-- 10000 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 96877 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 117 100 117 0 0 198k 0 --:--:-- --:--:-- --:--:-- 114k {"connections":0,"version":"8.0.11-TiDB-v7.5.1-43-ge406d5780b","git_hash":"e406d5780b18a1f2aaf6230cde4b6403991e228d"}process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D1885936C626 < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:38:36 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-master.test already exit Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-worker.test already exit [Fri Apr 26 19:38:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:38:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:38:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 63563 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D18D5E2645AC < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:38:58 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:39:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:39:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:39:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 64213 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test without check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D19412514C8C < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:39:27 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:39:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:39:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:39:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 64902 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with special task-name and check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D19AC7B9F504 < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:39:55 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:40:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:40:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:40:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 4-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 65576 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with special task-name and without check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17C9D1A1F7924D9E < X-Xss-Protection: 1; mode=block < Date: Fri, 26 Apr 2024 11:40:26 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Fri Apr 26 19:40:33 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:40:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:40:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml" error check wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 66272 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test error check success Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:40:53 CST 2024] <<<<<< test case s3_dumpling_lightning success! >>>>>> start running case: [sequence_sharding_optimistic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:40:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:40:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:40:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 341 100 216 100 125 62140 35960 --:--:-- --:--:-- --:--:-- 72000 dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_optimistic/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_optimistic/source2.yaml" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 407 100 282 100 125 61842 27412 --:--:-- --:--:-- --:--:-- 70500 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-task.yaml --remove-meta" check diff successfully ERROR 1146 (42S02) at line 1: Table 'sharding_seq_tmp.t1' doesn't exist run tidb sql failed 1-th time, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 398 100 273 100 125 73565 33683 --:--:-- --:--:-- --:--:-- 91000 dmctl test cmd: "pause-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=2 expected=2 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 512 100 387 100 125 106k 35063 --:--:-- --:--:-- --:--:-- 125k dmctl test cmd: "resume-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=3 expected=3 restart dm-worker1 wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit [Fri Apr 26 19:41:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "shard-ddl-lock unlock non-exist-task-`test_db`.`test_table`" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" got=3 expected=3 check diff successfully dmctl test cmd: "pause-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=2 expected=2 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 508 100 383 100 125 92045 30040 --:--:-- --:--:-- --:--:-- 95750 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 291 100 166 100 125 57618 43387 --:--:-- --:--:-- --:--:-- 83000 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 538 100 370 100 168 71469 32451 --:--:-- --:--:-- --:--:-- 74000 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 508 100 383 100 125 121k 40610 --:--:-- --:--:-- --:--:-- 124k dmctl test cmd: "binlog-schema list -s mysql-replica-01,mysql-replica-02 sequence_sharding_optimistic sharding_seq_opt t2" dmctl test cmd: "binlog-schema delete -s mysql-replica-01 sequence_sharding_optimistic sharding_seq_opt t2" dmctl test cmd: "binlog-schema update -s mysql-replica-01 sequence_sharding_optimistic sharding_seq_opt t1 /tmp/dm_test/sequence_sharding_optimistic/schema.sql" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 508 100 383 100 125 106k 35521 --:--:-- --:--:-- --:--:-- 124k { "result": true, "msg": "", "sources": [ { "result": true, "msg": "CREATE TABLE `t1` ( `id` bigint(20) NOT NULL, `c2` varchar(20) DEFAULT NULL, `c3` bigint(11) DEFAULT NULL, PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin", "source": "mysql-replica-01", "worker": "worker1" } ] }dmctl test cmd: "resume-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=3 expected=3 check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:41:12 CST 2024] <<<<<< test case sequence_sharding_optimistic success! >>>>>> start running case: [sequence_sharding_removemeta] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:41:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:41:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:41:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_removemeta/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_removemeta/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-task.yaml " check diff successfully dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status sequence_sharding_removemeta" got=0 expected=0 got=0 expected=0 dmctl test cmd: "stop-task sequence_sharding_removemeta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-task.yaml --remove-meta" dmctl test cmd: "shard-ddl-lock" got=1 expected=1 check diff successfully dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status sequence_sharding_removemeta" got=1 expected=1 dmctl test cmd: "stop-task sequence_sharding_removemeta" dmctl test cmd: "shard-ddl-lock unlock sequence_sharding_removemeta-`sharding_target3`.`t_target`" dmctl test cmd: "shard-ddl-lock" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:41:43 CST 2024] <<<<<< test case sequence_sharding_removemeta success! >>>>>> start running case: [shardddl_optimistic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:41:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:41:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl_optimistic/source1.yaml" [Fri Apr 26 19:41:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl_optimistic/source2.yaml" [Fri Apr 26 19:41:47 CST 2024] <<<<<< start DM-DIFFERENT_SCHEMA_FULL optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:41:49 CST 2024] <<<<<< finish DM-DIFFERENT_SCHEMA_FULL optimistic >>>>>> [Fri Apr 26 19:41:49 CST 2024] <<<<<< start DM-DIFFERENT_SCHEMA_INCREMENTAL optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "pause-task test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 245 100 245 0 0 79339 0 --:--:-- --:--:-- --:--:-- 119k % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 275 100 275 0 0 131k 0 --:--:-- --:--:-- --:--:-- 268k % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 268 100 268 0 0 110k 0 --:--:-- --:--:-- --:--:-- 130k % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 298 100 298 0 0 141k 0 --:--:-- --:--:-- --:--:-- 291k dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/shardddl_optimistic/task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d shardddl1 -t tb1 /tmp/dm_test/shardddl_optimistic/schema11.sql" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d shardddl1 -t tb2 /tmp/dm_test/shardddl_optimistic/schema12.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d shardddl1 -t tb1 /tmp/dm_test/shardddl_optimistic/schema21.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d shardddl1 -t tb2 /tmp/dm_test/shardddl_optimistic/schema22.sql" dmctl test cmd: "resume-task test" run tidb sql failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:41:56 CST 2024] <<<<<< finish DM-DIFFERENT_SCHEMA_INCREMENTAL optimistic >>>>>> [Fri Apr 26 19:41:56 CST 2024] <<<<<< start DM-RESTART_TASK_MASTER_WORKER optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later restart master restart dm-master wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Fri Apr 26 19:42:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later restart worker2 restart dm-worker2 wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:42:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later restart master restart dm-master wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Fri Apr 26 19:42:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later restart worker2 restart dm-worker2 wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:42:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:42:40 CST 2024] <<<<<< finish DM-RESTART_TASK_MASTER_WORKER optimistic >>>>>> [Fri Apr 26 19:42:40 CST 2024] <<<<<< start DM-STOP_TASK_FOR_A_SOURCE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test -s mysql-replica-02" run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml -s mysql-replica-02" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:42:48 CST 2024] <<<<<< finish DM-STOP_TASK_FOR_A_SOURCE optimistic >>>>>> [Fri Apr 26 19:42:48 CST 2024] <<<<<< start DM-UPDATE_BA_ROUTE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/shardddl_optimistic/task.yaml" dmctl test cmd: "show-ddl-locks" got=1 expected=1 run tidb sql failed 1-th time, retry later run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Fri Apr 26 19:42:59 CST 2024] <<<<<< finish DM-UPDATE_BA_ROUTE optimistic >>>>>> [Fri Apr 26 19:42:59 CST 2024] <<<<<< start DM-CREATE_DROP_TABLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later run tidb sql failed 1-th time, retry later dmctl test cmd: "show-ddl-locks" dmctl test cmd: "stop-task test" [Fri Apr 26 19:43:06 CST 2024] <<<<<< finish DM-CREATE_DROP_TABLE optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:43:11 CST 2024] <<<<<< test case shardddl_optimistic success! >>>>>> start running case: [slow_relay_writer] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:43:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:43:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/slow_relay_writer/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/conf/dm-task.yaml " check diff successfully start incremental_data finish incremental_data dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:43:31 CST 2024] <<<<<< test case slow_relay_writer success! >>>>>> start running case: [sql_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:43:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:43:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:43:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sql_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sql_mode/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-task.yaml " check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully ERROR 1146 (42S02) at line 1: Table 'sql_mode.t0' doesn't exist run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:43:45 CST 2024] <<<<<< test case sql_mode success! >>>>>> start running case: [sync_collation] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:43:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:43:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:43:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sync_collation/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sync_collation/source2.yaml" prepare data start task dmctl test cmd: "start-task /tmp/dm_test/sync_collation/dm-task.yaml --remove-meta" check full phase prepare incremental data check incremental phase run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task sync_collation" prepare data for full phase error test dmctl test cmd: "start-task /tmp/dm_test/sync_collation/dm-task.yaml --remove-meta" check full phase error dmctl test cmd: "query-status sync_collation" got=1 expected=1 dmctl test cmd: "stop-task sync_collation" prepare data for incremental phase error test dmctl test cmd: "start-task /tmp/dm_test/sync_collation/dm-task.yaml --remove-meta" check incremental phase error dmctl test cmd: "query-status sync_collation" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:44:02 CST 2024] <<<<<< test case sync_collation success! >>>>>> start running case: [tracker_ignored_ddl] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:44:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:44:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tracker_ignored_ddl/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-task.yaml " check diff successfully increment1 check success dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "resume-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 increment2 check success dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/tracker_ignored_ddl/source1.yaml" [Fri Apr 26 19:44:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-master.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:44:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-worker.toml >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tracker_ignored_ddl/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-task.yaml " check diff successfully increment1 check success dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "resume-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 increment2 check success dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/tracker_ignored_ddl/source1.yaml" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:44:21 CST 2024] <<<<<< test case tracker_ignored_ddl success! >>>>>> start running case: [validator_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh... Verbose mode = false --> full mode, check we validate different data types(gtid=false, relay=false) 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:44:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:44:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:44:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=false, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:44:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:44:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:44:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:44:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:44:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:44:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:45:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:45:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:45:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> fast mode, check we validate different data types 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:45:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:45:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:45:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: full mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:45:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:45:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:45:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: fast mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:45:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:45:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:45:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check update pk(split into insert and delete) dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 7\/1\/2" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator panic and we can catch it 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:46:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:46:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:46:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test validator panic count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 --> check validator worker panic and we can catch it 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:46:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:46:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:46:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator stop when pending row size too large 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:46:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:46:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:46:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1+ got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator stop when pending row count too many 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:46:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:46:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:46:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1+ got=1 expected=1 got=1 expected=1 got=1 expected=1 --> start validator on the fly, validate from current syncer progress(gtid=false, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:47:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:47:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:47:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=false, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:47:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:47:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:47:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=true, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:47:42 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:47:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:47:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=true, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:48:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:48:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:48:04 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time < min mysql binlog pos(gtid=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:48:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:48:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:48:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time < min mysql binlog pos(gtid=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:48:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:48:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:48:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time which is in range of mysql binlog(gtid=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:48:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:48:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:48:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2024-04-26 13:49:03' test" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time which is in range of mysql binlog(gtid=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:49:16 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:49:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:49:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2024-04-26 13:49:24' test" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time > max mysql binlog pos 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:49:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:49:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:49:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2026-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> single varchar col pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:49:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:49:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:49:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> single datetime col pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:50:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:50:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:50:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> compound pk (datetime, timestamp, int, varchar) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:50:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:50:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:50:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/4\/3" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table without primary key 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:50:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:50:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:50:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table is deleted on downstream 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:50:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:50:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:50:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table in schema-tracker has less column than binlog 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:51:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:51:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:51:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> pk column of downstream table not in range of binlog column 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:51:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:51:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:51:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> stopped validator fail over 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:51:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:51:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:51:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "validation stop test" got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process worker1 exit... wait process worker1 exit... process worker1 already exit dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation start test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/3\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> filter online ddl shadow table 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:52:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:52:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:52:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/test-filter.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> filter by ba list dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> filter by filter-rules dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 7\/2\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=3 expected=3 --> validate when syncer is stopped 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:52:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:52:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:52:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 --> test duplicate auto-incr pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:54:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:54:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:54:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/sharding-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=2 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 1 != expected: 2, failed the 0-th time, will retry again got=2 expected=2 got=2 expected=2 got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:54:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:54:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Fri Apr 26 19:54:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-long-interval.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-gtid fe276a06-03bd-11ef-b264-0601375cf6a9:1-79" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=0 expected=1 command: query-status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-gtid fe276a06-03bd-11ef-b264-0601375cf6a9:1-81" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:55:04 CST 2024] <<<<<< test case validator_basic success! >>>>>> restore config restore time_zone start running case: [validator_checkpoint] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh... Verbose mode = false --> check persist checkpoint and data with 2 source 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:55:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:55:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:55:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta" dmctl test cmd: ""unit": "Sync"" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test processedRowsStatus": "insert\/update\/delete: 3\/1\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> check validator can restart from previous position on fail over restart dm-worker1 wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Fri Apr 26 19:55:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive restart dm-worker2 wait process worker2 exit... wait process worker2 exit... process worker2 already exit [Fri Apr 26 19:55:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later --> check validator persist fail in the middle 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:55:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:55:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Fri Apr 26 19:55:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta" dmctl test cmd: ""unit": "Sync"" dmctl test cmd: "validation status test" got=0 expected=2 command: validation status test "stage": "Stopped" count: 0 != expected: 2, failed the 0-th time, will retry again got=2 expected=2 got=2 expected=2 got=2 expected=2 got=2 expected=2 got=2 expected=2 --> after resume and persist again, meta data should be consistent dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml " dmctl test cmd: "validation status test" got=4 expected=4 got=2 expected=2 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later --> check validate success after insert data manually dmctl test cmd: "validation status test" got=4 expected=4 got=2 expected=2 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again got=4 expected=4 got=2 expected=2 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:55:54 CST 2024] <<<<<< test case validator_checkpoint success! >>>>>> start running case: [incompatible_ddl_changes] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Fri Apr 26 19:55:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Fri Apr 26 19:55:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 6-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 7-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 8-th time, will retry again got=0 expected=1 command: query-status test event modify constaints count: 0 != expected: 1, failed the 9-th time, will retry again { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-01", "worker": "worker1", "result": null, "relayStatus": null }, "subTaskStatus": [ { "name": "test", "stage": "Paused", "unit": "Sync", "result": { "isCanceled": false, "errors": [ { "ErrCode": 36015, "ErrClass": "sync-unit", "ErrScope": "internal", "ErrLevel": "high", "Message": "startLocation: [position: (dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 17045), gtid-set: fe276a06-03bd-11ef-b264-0601375cf6a9:1-47], endLocation: [position: (dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 17220), gtid-set: fe276a06-03bd-11ef-b264-0601375cf6a9:1-48], origin SQL: [alter table incompatible_ddl_changes.t1 add constraint c_int_unique unique(c_int)]: event modify constraint on `incompatible_ddl_changes`.`t1`", "RawCause": "", "Workaround": "" } ], "detail": null }, "unresolvedDDLLockID": "", "sync": { "totalEvents": "102", "totalTps": "25", "recentTps": "0", "masterBinlog": "(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 17220)", "masterBinlogGtid": "fe276a06-03bd-11ef-b264-0601375cf6a9:1-48", "syncerBinlog": "(dm-it-d9e3c32b-1a55-4ba7-9921-a09c98cc1680-q6qxw-x9kv2-bin.000001, 16980)", "syncerBinlogGtid": "fe276a06-03bd-11ef-b264-0601375cf6a9:1-47", "blockingDDLs": [ ], "unresolvedGroups": [ ], "synced": false, "binlogType": "remote", "secondsBehindMaster": "0", "blockDDLOwner": "", "conflictMsg": "", "totalRows": "102", "totalRps": "25", "recentRps": "0" }, "validation": null } ] } ] } PASS coverage: 3.5% of statements in github.com/pingcap/tiflow/dm/... curl: (7) Failed connect to 127.0.0.1:8361; Connection refused curl: (7) Failed connect to 127.0.0.1:8461; Connection refused curl: (7) Failed connect to 127.0.0.1:8561; Connection refused curl: (7) Failed connect to 127.0.0.1:8661; Connection refused curl: (7) Failed connect to 127.0.0.1:8761; Connection refused curl: (7) Failed connect to 127.0.0.1:8263; Connection refused curl: (7) Failed connect to 127.0.0.1:8264; Connection refused curl: (7) Failed connect to 127.0.0.1:18262; Connection refused curl: (7) Failed connect to 127.0.0.1:18263; Connection refused make: *** [dm_integration_test_in_group] Error 1 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Post stage [Pipeline] sh + ls /tmp/dm_test adjust_gtid async_checkpoint_flush binlog_parse case_sensitive check_task checkpoint_transaction cov.adjust_gtid.dmctl.1714130189.657.out cov.adjust_gtid.dmctl.1714130191.774.out cov.adjust_gtid.dmctl.1714130192.821.out cov.adjust_gtid.dmctl.1714130193.962.out cov.adjust_gtid.dmctl.1714130203.1006.out cov.adjust_gtid.dmctl.1714130205.1226.out cov.adjust_gtid.master.out cov.adjust_gtid.worker.8262.1714130188.out cov.adjust_gtid.worker.8262.1714130203.out cov.adjust_gtid.worker.8262.1714130208.out cov.adjust_gtid.worker.8263.1714130189.out cov.adjust_gtid.worker.8263.1714130204.out cov.adjust_gtid.worker.8263.1714130209.out cov.async_checkpoint_flush.dmctl.1714130217.1859.out cov.async_checkpoint_flush.dmctl.1714130217.1909.out cov.async_checkpoint_flush.master.out cov.async_checkpoint_flush.worker.8262.1714130216.out cov.binlog_parse.dmctl.1714130256.16413.out cov.binlog_parse.dmctl.1714130257.16460.out cov.binlog_parse.dmctl.1714130258.16544.out cov.binlog_parse.dmctl.1714130259.16617.out cov.binlog_parse.dmctl.1714130260.16662.out cov.binlog_parse.master.out cov.binlog_parse.worker.8262.1714130255.out cov.case_sensitive.dmctl.1714130267.17081.out cov.case_sensitive.dmctl.1714130269.17198.out cov.case_sensitive.dmctl.1714130270.17238.out cov.case_sensitive.dmctl.1714130274.17455.out cov.case_sensitive.dmctl.1714130291.17740.out cov.case_sensitive.dmctl.1714130291.17787.out cov.case_sensitive.dmctl.1714130291.17828.out cov.case_sensitive.dmctl.1714130299.18175.out cov.case_sensitive.dmctl.1714130301.18300.out cov.case_sensitive.dmctl.1714130302.18338.out cov.case_sensitive.dmctl.1714130308.18588.out cov.case_sensitive.dmctl.1714130325.18879.out cov.case_sensitive.dmctl.1714130325.18926.out cov.case_sensitive.dmctl.1714130325.18967.out cov.case_sensitive.master.out cov.case_sensitive.worker.8262.1714130266.out cov.case_sensitive.worker.8262.1714130273.out cov.case_sensitive.worker.8262.1714130298.out cov.case_sensitive.worker.8262.1714130307.out cov.case_sensitive.worker.8263.1714130268.out cov.case_sensitive.worker.8263.1714130275.out cov.case_sensitive.worker.8263.1714130299.out cov.case_sensitive.worker.8263.1714130310.out cov.check_task.dmctl.1714130406.20632.out cov.check_task.dmctl.1714130407.20848.out cov.check_task.dmctl.1714130407.20906.out cov.check_task.dmctl.1714130407.20979.out cov.check_task.dmctl.1714130410.21089.out cov.check_task.dmctl.1714130410.21159.out cov.check_task.dmctl.1714130410.21219.out cov.check_task.dmctl.1714130410.21269.out cov.check_task.dmctl.1714130410.21325.out cov.check_task.master.out cov.check_task.worker.8262.1714130405.out cov.checkpoint_transaction.dmctl.1714130333.19339.out cov.checkpoint_transaction.dmctl.1714130334.19384.out cov.checkpoint_transaction.dmctl.1714130364.19890.out cov.checkpoint_transaction.dmctl.1714130379.19939.out cov.checkpoint_transaction.dmctl.1714130379.20010.out cov.checkpoint_transaction.dmctl.1714130379.20051.out cov.checkpoint_transaction.dmctl.1714130382.20187.out cov.checkpoint_transaction.dmctl.1714130382.20232.out cov.checkpoint_transaction.master.out cov.checkpoint_transaction.worker.8262.1714130332.out cov.checkpoint_transaction.worker.8262.1714130381.out cov.dm_syncer.dmctl.1714130419.21779.out cov.dm_syncer.dmctl.1714130419.21821.out cov.dm_syncer.dmctl.1714130420.21867.out cov.dm_syncer.dmctl.1714130422.21968.out cov.dm_syncer.master.out cov.dm_syncer.syncer.out cov.dm_syncer.worker.8262.1714130417.out cov.dm_syncer.worker.8263.1714130418.out cov.downstream_diff_index.dmctl.1714130439.22569.out cov.downstream_diff_index.dmctl.1714130440.22688.out cov.downstream_diff_index.dmctl.1714130441.22726.out cov.downstream_diff_index.master.out cov.downstream_diff_index.worker.8262.1714130438.out cov.downstream_diff_index.worker.8263.1714130439.out cov.downstream_more_column.dmctl.1714130452.23164.out cov.downstream_more_column.dmctl.1714130453.23206.out cov.downstream_more_column.dmctl.1714130456.23280.out cov.downstream_more_column.dmctl.1714130456.23320.out cov.downstream_more_column.dmctl.1714130458.23370.out cov.downstream_more_column.dmctl.1714130458.23414.out cov.downstream_more_column.dmctl.1714130458.23452.out cov.downstream_more_column.dmctl.1714130458.23487.out cov.downstream_more_column.master.out cov.downstream_more_column.worker.8262.1714130451.out cov.drop_column_with_index.dmctl.1714130467.23850.out cov.drop_column_with_index.dmctl.1714130468.23892.out cov.drop_column_with_index.dmctl.1714130469.23935.out cov.drop_column_with_index.dmctl.1714130469.24011.out cov.drop_column_with_index.dmctl.1714130469.24054.out cov.drop_column_with_index.dmctl.1714130470.24098.out cov.drop_column_with_index.dmctl.1714130475.24152.out cov.drop_column_with_index.dmctl.1714130475.24228.out cov.drop_column_with_index.master.out cov.drop_column_with_index.worker.8262.1714130466.out cov.duplicate_event.dmctl.1714130485.24635.out cov.duplicate_event.dmctl.1714130486.24679.out cov.duplicate_event.dmctl.1714130550.25023.out cov.duplicate_event.dmctl.1714130551.25074.out cov.duplicate_event.dmctl.1714130553.25151.out cov.duplicate_event.dmctl.1714130554.25197.out cov.duplicate_event.dmctl.1714130575.25767.out cov.duplicate_event.dmctl.1714130575.25808.out cov.duplicate_event.dmctl.1714130640.26146.out cov.duplicate_event.dmctl.1714130641.26194.out cov.duplicate_event.dmctl.1714130642.26275.out cov.duplicate_event.dmctl.1714130643.26320.out cov.duplicate_event.master.out cov.duplicate_event.worker.8262.1714130484.out cov.duplicate_event.worker.8262.1714130574.out cov.duplicate_event.worker.8263.1714130549.out cov.duplicate_event.worker.8263.1714130639.out cov.expression_filter.dmctl.1714130651.26702.out cov.expression_filter.dmctl.1714130651.26746.out cov.expression_filter.dmctl.1714130652.26789.out cov.expression_filter.dmctl.1714130652.26831.out cov.expression_filter.dmctl.1714130652.26868.out cov.expression_filter.dmctl.1714130654.26918.out cov.expression_filter.dmctl.1714130656.27086.out cov.expression_filter.dmctl.1714130663.27367.out cov.expression_filter.dmctl.1714130664.27416.out cov.expression_filter.dmctl.1714130670.27473.out cov.expression_filter.master.out cov.expression_filter.worker.8262.1714130650.out cov.expression_filter.worker.8262.1714130662.out cov.extend_column.dmctl.1714130679.27921.out cov.extend_column.dmctl.1714130679.27964.out cov.extend_column.dmctl.1714130680.28018.out cov.extend_column.dmctl.1714130680.28095.out cov.extend_column.dmctl.1714130692.28713.out cov.extend_column.dmctl.1714130692.28755.out cov.extend_column.dmctl.1714130694.28809.out cov.extend_column.dmctl.1714130694.28856.out cov.extend_column.master.out cov.extend_column.worker.8262.1714130676.out cov.extend_column.worker.8262.1714130690.out cov.extend_column.worker.8263.1714130677.out cov.extend_column.worker.8263.1714130691.out cov.fake_rotate_event.dmctl.1714130705.29420.out cov.fake_rotate_event.dmctl.1714130705.29463.out cov.fake_rotate_event.dmctl.1714130706.29511.out cov.fake_rotate_event.dmctl.1714130710.29750.out cov.fake_rotate_event.master.out cov.fake_rotate_event.worker.8262.1714130704.out cov.fake_rotate_event.worker.8262.1714130708.out cov.foreign_key.dmctl.1714130718.30118.out cov.foreign_key.dmctl.1714130718.30158.out cov.foreign_key.dmctl.1714130720.30214.out cov.foreign_key.dmctl.1714130720.30284.out cov.foreign_key.master.out cov.foreign_key.worker.8262.1714130717.out cov.full_mode.dmctl.1714130729.30686.out cov.full_mode.dmctl.1714130730.30807.out cov.full_mode.dmctl.1714130731.30849.out cov.full_mode.dmctl.1714130733.30905.out cov.full_mode.dmctl.1714130742.31337.out cov.full_mode.dmctl.1714130742.31379.out cov.full_mode.dmctl.1714130743.31421.out cov.full_mode.dmctl.1714130754.31859.out cov.full_mode.dmctl.1714130755.31981.out cov.full_mode.dmctl.1714130757.32022.out cov.full_mode.dmctl.1714130759.32099.out cov.full_mode.dmctl.1714130767.32436.out cov.full_mode.dmctl.1714130769.32549.out cov.full_mode.dmctl.1714130771.32590.out cov.full_mode.dmctl.1714130772.32631.out cov.full_mode.dmctl.1714130782.33107.out cov.full_mode.dmctl.1714130783.33157.out cov.full_mode.dmctl.1714130784.33202.out cov.full_mode.dmctl.1714130785.33262.out cov.full_mode.master.out cov.full_mode.worker.8262.1714130728.out cov.full_mode.worker.8262.1714130740.out cov.full_mode.worker.8262.1714130753.out cov.full_mode.worker.8262.1714130766.out cov.full_mode.worker.8262.1714130779.out cov.full_mode.worker.8263.1714130729.out cov.full_mode.worker.8263.1714130741.out cov.full_mode.worker.8263.1714130754.out cov.full_mode.worker.8263.1714130768.out cov.full_mode.worker.8263.1714130781.out cov.gbk.dmctl.1714130795.33781.out cov.gbk.dmctl.1714130795.33821.out cov.gbk.dmctl.1714130796.33872.out cov.gbk.dmctl.1714130833.35328.out cov.gbk.dmctl.1714130836.35414.out cov.gbk.dmctl.1714130865.36696.out cov.gbk.master.out cov.gbk.worker.8262.1714130793.out cov.gbk.worker.8262.1714130812.out cov.gbk.worker.8262.1714130817.out cov.gbk.worker.8262.1714130822.out cov.gbk.worker.8262.1714130827.out cov.gbk.worker.8262.1714130831.out cov.gbk.worker.8262.1714130838.out cov.gbk.worker.8262.1714130844.out cov.gbk.worker.8262.1714130850.out cov.gbk.worker.8262.1714130857.out cov.gbk.worker.8262.1714130863.out cov.gbk.worker.8263.1714130794.out cov.gbk.worker.8263.1714130813.out cov.gbk.worker.8263.1714130818.out cov.gbk.worker.8263.1714130823.out cov.gbk.worker.8263.1714130828.out cov.gbk.worker.8263.1714130832.out cov.gbk.worker.8263.1714130839.out cov.gbk.worker.8263.1714130845.out cov.gbk.worker.8263.1714130852.out cov.gbk.worker.8263.1714130858.out cov.gbk.worker.8263.1714130864.out cov.gtid.dmctl.1714130875.37077.out cov.gtid.dmctl.1714130876.37197.out cov.gtid.dmctl.1714130877.37237.out cov.gtid.dmctl.1714130879.37351.out cov.gtid.dmctl.1714130879.37394.out cov.gtid.dmctl.1714130888.37776.out cov.gtid.dmctl.1714130889.37893.out cov.gtid.dmctl.1714130890.37935.out cov.gtid.dmctl.1714130892.38126.out cov.gtid.dmctl.1714130893.38322.out cov.gtid.dmctl.1714130894.38416.out cov.gtid.dmctl.1714130897.38636.out cov.gtid.dmctl.1714130899.38680.out cov.gtid.master.out cov.gtid.worker.8262.1714130874.out cov.gtid.worker.8262.1714130887.out cov.gtid.worker.8262.1714130892.out cov.gtid.worker.8262.1714130896.out cov.gtid.worker.8263.1714130875.out cov.gtid.worker.8263.1714130888.out cov.gtid.worker.8263.1714130892.out cov.gtid.worker.8263.1714130896.out cov.ha_cases.dmctl.1714130907.39071.out cov.ha_cases.dmctl.1714130909.39188.out cov.ha_cases.dmctl.1714130910.39227.out cov.ha_cases.dmctl.1714130910.39269.out cov.ha_cases.dmctl.1714130910.39306.out cov.ha_cases.dmctl.1714130920.39640.out cov.ha_cases.dmctl.1714130922.39763.out cov.ha_cases.dmctl.1714130923.39806.out cov.ha_cases.dmctl.1714130924.39844.out cov.ha_cases.dmctl.1714130928.39989.out cov.ha_cases.dmctl.1714130929.40111.out cov.ha_cases.dmctl.1714130930.40149.out cov.ha_cases.dmctl.1714130932.40216.out cov.ha_cases.dmctl.1714130933.40333.out cov.ha_cases.dmctl.1714130947.40895.out cov.ha_cases.dmctl.1714130950.41017.out cov.ha_cases.dmctl.1714130951.41054.out cov.ha_cases.dmctl.1714130952.41130.out cov.ha_cases.dmctl.1714130956.41341.out cov.ha_cases.dmctl.1714130957.41389.out cov.ha_cases.dmctl.1714130959.41436.out cov.ha_cases.dmctl.1714130963.41523.out cov.ha_cases.dmctl.1714130965.41753.out cov.ha_cases.dmctl.1714130970.41929.out cov.ha_cases.dmctl.1714130972.42131.out cov.ha_cases.dmctl.1714130977.42306.out cov.ha_cases.dmctl.1714130979.42531.out cov.ha_cases.dmctl.1714130980.42645.out cov.ha_cases.dmctl.1714130981.42686.out cov.ha_cases.dmctl.1714130982.42732.out cov.ha_cases.dmctl.1714130987.42880.out cov.ha_cases.dmctl.1714130989.43093.out cov.ha_cases.dmctl.1714131007.43739.out cov.ha_cases.dmctl.1714131036.44186.out cov.ha_cases.dmctl.1714131036.44225.out cov.ha_cases.dmctl.1714131036.44265.out cov.ha_cases.dmctl.1714131037.44305.out cov.ha_cases.dmctl.1714131038.44423.out cov.ha_cases.dmctl.1714131041.44500.out cov.ha_cases.dmctl.1714131045.44685.out cov.ha_cases.dmctl.1714131063.45187.out cov.ha_cases.dmctl.1714131064.45228.out cov.ha_cases.dmctl.1714131067.45381.out cov.ha_cases.dmctl.1714131068.45423.out cov.ha_cases.dmctl.1714131069.45558.out cov.ha_cases.dmctl.1714131070.45644.out cov.ha_cases.dmctl.1714131072.45708.out cov.ha_cases.dmctl.1714131072.45747.out cov.ha_cases.dmctl.1714131072.45787.out cov.ha_cases.dmctl.1714131073.45824.out cov.ha_cases.master.out cov.ha_cases.worker.18262.1714130978.out cov.ha_cases.worker.8262.1714130906.out cov.ha_cases.worker.8262.1714130919.out cov.ha_cases.worker.8262.1714130928.out cov.ha_cases.worker.8262.1714130946.out cov.ha_cases.worker.8262.1714130963.out cov.ha_cases.worker.8262.1714130971.out cov.ha_cases.worker.8262.1714130987.out cov.ha_cases.worker.8262.1714131041.out cov.ha_cases.worker.8262.1714131062.out cov.ha_cases.worker.8263.1714130907.out cov.ha_cases.worker.8263.1714130921.out cov.ha_cases.worker.8263.1714130932.out cov.ha_cases.worker.8263.1714130949.out cov.ha_cases.worker.8263.1714130964.out cov.ha_cases.worker.8263.1714130970.out cov.ha_cases.worker.8263.1714130988.out cov.ha_cases.worker.8263.1714131037.out cov.ha_cases.worker.8263.1714131068.out cov.ha_cases.worker.8264.1714130925.out cov.ha_cases.worker.8264.1714130977.out cov.http_proxies.dmctl.1714131084.46210.out cov.http_proxies.dmctl.1714131095.46514.out cov.http_proxies.dmctl.1714131106.46822.out cov.http_proxies.master.out cov.http_proxies.worker.8262.1714131083.out cov.http_proxies.worker.8262.1714131094.out cov.http_proxies.worker.8262.1714131105.out cov.incompatible_ddl_changes.dmctl.1714132556.105578.out cov.incompatible_ddl_changes.dmctl.1714132557.105624.out cov.incompatible_ddl_changes.dmctl.1714132558.105686.out cov.incompatible_ddl_changes.dmctl.1714132560.105767.out cov.incompatible_ddl_changes.dmctl.1714132560.105804.out cov.incompatible_ddl_changes.dmctl.1714132560.105853.out cov.incompatible_ddl_changes.dmctl.1714132561.105893.out cov.incompatible_ddl_changes.dmctl.1714132561.105939.out cov.incompatible_ddl_changes.dmctl.1714132561.105985.out cov.incompatible_ddl_changes.dmctl.1714132561.106027.out cov.incompatible_ddl_changes.dmctl.1714132561.106074.out cov.incompatible_ddl_changes.dmctl.1714132561.106120.out cov.incompatible_ddl_changes.dmctl.1714132561.106154.out cov.incompatible_ddl_changes.dmctl.1714132561.106198.out cov.incompatible_ddl_changes.dmctl.1714132562.106240.out cov.incompatible_ddl_changes.dmctl.1714132562.106286.out cov.incompatible_ddl_changes.dmctl.1714132562.106324.out cov.incompatible_ddl_changes.dmctl.1714132562.106367.out cov.incompatible_ddl_changes.dmctl.1714132562.106407.out cov.incompatible_ddl_changes.dmctl.1714132562.106453.out cov.incompatible_ddl_changes.dmctl.1714132562.106490.out cov.incompatible_ddl_changes.dmctl.1714132562.106532.out cov.incompatible_ddl_changes.dmctl.1714132563.106566.out cov.incompatible_ddl_changes.dmctl.1714132563.106624.out cov.incompatible_ddl_changes.dmctl.1714132563.106665.out cov.incompatible_ddl_changes.dmctl.1714132563.106710.out cov.lightning_load_task.dmctl.1714131115.47222.out cov.lightning_load_task.dmctl.1714131117.47340.out cov.lightning_load_task.dmctl.1714131119.47467.out cov.lightning_load_task.dmctl.1714131120.47528.out cov.lightning_load_task.dmctl.1714131121.47612.out cov.lightning_load_task.dmctl.1714131121.47656.out cov.lightning_load_task.dmctl.1714131124.47730.out cov.lightning_load_task.dmctl.1714131124.47776.out cov.lightning_load_task.dmctl.1714131124.47816.out cov.lightning_load_task.dmctl.1714131125.47953.out cov.lightning_load_task.dmctl.1714131125.47993.out cov.lightning_load_task.dmctl.1714131125.48036.out cov.lightning_load_task.dmctl.1714131125.48079.out cov.lightning_load_task.dmctl.1714131128.48152.out cov.lightning_load_task.dmctl.1714131128.48197.out cov.lightning_load_task.dmctl.1714131128.48236.out cov.lightning_load_task.dmctl.1714131129.48284.out cov.lightning_load_task.dmctl.1714131130.48396.out cov.lightning_load_task.dmctl.1714131132.48467.out cov.lightning_load_task.dmctl.1714131132.48509.out cov.lightning_load_task.dmctl.1714131134.48553.out cov.lightning_load_task.dmctl.1714131135.48666.out cov.lightning_load_task.dmctl.1714131137.48746.out cov.lightning_load_task.dmctl.1714131137.48790.out cov.lightning_load_task.dmctl.1714131137.48828.out cov.lightning_load_task.dmctl.1714131137.48867.out cov.lightning_load_task.dmctl.1714131148.49116.out cov.lightning_load_task.dmctl.1714131148.49166.out cov.lightning_load_task.dmctl.1714131149.49294.out cov.lightning_load_task.dmctl.1714131150.49337.out cov.lightning_load_task.master.out cov.lightning_load_task.worker.8262.1714131114.out cov.lightning_load_task.worker.8262.1714131124.out cov.lightning_load_task.worker.8262.1714131134.out cov.lightning_load_task.worker.8263.1714131116.out cov.lightning_load_task.worker.8263.1714131129.out cov.lightning_load_task.worker.8264.1714131118.out cov.lightning_load_task.worker.8264.1714131148.out cov.lightning_mode.dmctl.1714131188.50439.out cov.lightning_mode.dmctl.1714131189.50564.out cov.lightning_mode.dmctl.1714131190.50607.out cov.lightning_mode.dmctl.1714131195.50800.out cov.lightning_mode.dmctl.1714131197.50874.out cov.lightning_mode.dmctl.1714131203.51058.out cov.lightning_mode.dmctl.1714131203.51092.out cov.lightning_mode.dmctl.1714131210.51245.out cov.lightning_mode.dmctl.1714131210.51292.out cov.lightning_mode.dmctl.1714131212.51338.out cov.lightning_mode.dmctl.1714131219.51520.out cov.lightning_mode.dmctl.1714131219.51554.out cov.lightning_mode.dmctl.1714131225.51714.out cov.lightning_mode.dmctl.1714131226.51758.out cov.lightning_mode.master.out cov.lightning_mode.worker.8262.1714131187.out cov.lightning_mode.worker.8263.1714131188.out cov.metrics.dmctl.1714131252.52504.out cov.metrics.dmctl.1714131254.52633.out cov.metrics.dmctl.1714131255.52678.out cov.metrics.dmctl.1714131256.52738.out cov.metrics.dmctl.1714131260.52879.out cov.metrics.dmctl.1714131277.53587.out cov.metrics.dmctl.1714131280.53778.out cov.metrics.master.out cov.metrics.worker.8262.1714131251.out cov.metrics.worker.8262.1714131262.out cov.metrics.worker.8262.1714131275.out cov.metrics.worker.8262.1714131279.out cov.metrics.worker.8263.1714131253.out cov.metrics.worker.8263.1714131263.out cov.metrics.worker.8263.1714131276.out cov.new_collation_off.dmctl.1714131292.54171.out cov.new_collation_off.dmctl.1714131292.54216.out cov.new_collation_off.master.out cov.new_collation_off.worker.8262.1714131291.out cov.only_dml.dmctl.1714131301.54611.out cov.only_dml.dmctl.1714131302.54730.out cov.only_dml.dmctl.1714131304.54783.out cov.only_dml.dmctl.1714131305.54896.out cov.only_dml.dmctl.1714131306.54972.out cov.only_dml.dmctl.1714131307.55042.out cov.only_dml.dmctl.1714131308.55113.out cov.only_dml.dmctl.1714131309.55189.out cov.only_dml.dmctl.1714131311.55259.out cov.only_dml.dmctl.1714131312.55329.out cov.only_dml.dmctl.1714131313.55400.out cov.only_dml.dmctl.1714131314.55478.out cov.only_dml.dmctl.1714131315.55560.out cov.only_dml.dmctl.1714131316.55643.out cov.only_dml.dmctl.1714131317.55716.out cov.only_dml.dmctl.1714131319.55792.out cov.only_dml.dmctl.1714131320.55862.out cov.only_dml.dmctl.1714131321.55935.out cov.only_dml.dmctl.1714131322.56005.out cov.only_dml.dmctl.1714131323.56081.out cov.only_dml.dmctl.1714131324.56154.out cov.only_dml.dmctl.1714131325.56223.out cov.only_dml.dmctl.1714131326.56293.out cov.only_dml.dmctl.1714131328.56378.out cov.only_dml.dmctl.1714131329.56449.out cov.only_dml.dmctl.1714131330.56521.out cov.only_dml.dmctl.1714131331.56595.out cov.only_dml.dmctl.1714131332.56670.out cov.only_dml.dmctl.1714131333.56742.out cov.only_dml.dmctl.1714131334.56814.out cov.only_dml.dmctl.1714131336.56886.out cov.only_dml.dmctl.1714131337.56957.out cov.only_dml.dmctl.1714131338.57027.out cov.only_dml.dmctl.1714131339.57132.out cov.only_dml.dmctl.1714131340.57207.out cov.only_dml.dmctl.1714131341.57280.out cov.only_dml.dmctl.1714131342.57355.out cov.only_dml.dmctl.1714131343.57424.out cov.only_dml.dmctl.1714131345.57512.out cov.only_dml.master.out cov.only_dml.worker.8262.1714131300.out cov.only_dml.worker.8263.1714131301.out cov.openapi.dmctl.1714131365.58115.out cov.openapi.dmctl.1714131365.58164.out cov.openapi.dmctl.1714131366.58211.out cov.openapi.dmctl.1714131367.58255.out cov.openapi.dmctl.1714131368.58321.out cov.openapi.dmctl.1714131370.58418.out cov.openapi.dmctl.1714131370.58470.out cov.openapi.dmctl.1714131372.58631.out cov.openapi.dmctl.1714131372.58688.out cov.openapi.dmctl.1714131373.58737.out cov.openapi.dmctl.1714131374.58790.out cov.openapi.dmctl.1714131377.58906.out cov.openapi.dmctl.1714131378.58945.out cov.openapi.dmctl.1714131381.59100.out cov.openapi.dmctl.1714131381.59151.out cov.openapi.dmctl.1714131381.59195.out cov.openapi.dmctl.1714131381.59230.out cov.openapi.dmctl.1714131387.59504.out cov.openapi.dmctl.1714131387.59557.out cov.openapi.dmctl.1714131393.59845.out cov.openapi.dmctl.1714131393.59895.out cov.openapi.dmctl.1714131395.60067.out cov.openapi.dmctl.1714131396.60167.out cov.openapi.dmctl.1714131397.60249.out cov.openapi.dmctl.1714131397.60325.out cov.openapi.dmctl.1714131399.60407.out cov.openapi.dmctl.1714131406.60572.out cov.openapi.dmctl.1714131407.60655.out cov.openapi.dmctl.1714131408.60696.out cov.openapi.dmctl.1714131409.60799.out cov.openapi.dmctl.1714131413.60854.out cov.openapi.dmctl.1714131414.60938.out cov.openapi.dmctl.1714131418.60992.out cov.openapi.dmctl.1714131419.61082.out cov.openapi.dmctl.1714131423.61145.out cov.openapi.dmctl.1714131432.61456.out cov.openapi.dmctl.1714131433.61508.out cov.openapi.dmctl.1714131441.61785.out cov.openapi.dmctl.1714131441.61843.out cov.openapi.master.out cov.openapi.worker.8262.1714131363.out cov.openapi.worker.8262.1714131384.out cov.openapi.worker.8262.1714131390.out cov.openapi.worker.8262.1714131426.out cov.openapi.worker.8262.1714131443.out cov.openapi.worker.8262.1714131465.out cov.openapi.worker.8262.1714131484.out cov.openapi.worker.8263.1714131364.out cov.openapi.worker.8263.1714131385.out cov.openapi.worker.8263.1714131391.out cov.openapi.worker.8263.1714131427.out cov.openapi.worker.8263.1714131444.out cov.openapi.worker.8263.1714131466.out cov.openapi.worker.8263.1714131485.out cov.s3_dumpling_lightning.dmctl.1714131519.63900.out cov.s3_dumpling_lightning.dmctl.1714131520.63942.out cov.s3_dumpling_lightning.dmctl.1714131521.64006.out cov.s3_dumpling_lightning.dmctl.1714131548.64586.out cov.s3_dumpling_lightning.dmctl.1714131548.64626.out cov.s3_dumpling_lightning.dmctl.1714131550.64694.out cov.s3_dumpling_lightning.dmctl.1714131577.65274.out cov.s3_dumpling_lightning.dmctl.1714131577.65316.out cov.s3_dumpling_lightning.dmctl.1714131578.65384.out cov.s3_dumpling_lightning.dmctl.1714131606.65960.out cov.s3_dumpling_lightning.dmctl.1714131606.66002.out cov.s3_dumpling_lightning.dmctl.1714131607.66068.out cov.s3_dumpling_lightning.dmctl.1714131637.66648.out cov.s3_dumpling_lightning.dmctl.1714131637.66692.out cov.s3_dumpling_lightning.dmctl.1714131638.66753.out cov.s3_dumpling_lightning.master.out cov.s3_dumpling_lightning.worker.8262.1714131517.out cov.s3_dumpling_lightning.worker.8262.1714131546.out cov.s3_dumpling_lightning.worker.8262.1714131575.out cov.s3_dumpling_lightning.worker.8262.1714131604.out cov.s3_dumpling_lightning.worker.8263.1714131518.out cov.s3_dumpling_lightning.worker.8263.1714131547.out cov.s3_dumpling_lightning.worker.8263.1714131576.out cov.s3_dumpling_lightning.worker.8263.1714131605.out cov.sequence_sharding_optimistic.dmctl.1714131656.67280.out cov.sequence_sharding_optimistic.dmctl.1714131656.67323.out cov.sequence_sharding_optimistic.dmctl.1714131657.67374.out cov.sequence_sharding_optimistic.dmctl.1714131661.67493.out cov.sequence_sharding_optimistic.dmctl.1714131661.67529.out cov.sequence_sharding_optimistic.dmctl.1714131661.67572.out cov.sequence_sharding_optimistic.dmctl.1714131661.67611.out cov.sequence_sharding_optimistic.dmctl.1714131664.67758.out cov.sequence_sharding_optimistic.dmctl.1714131665.67809.out cov.sequence_sharding_optimistic.dmctl.1714131665.67846.out cov.sequence_sharding_optimistic.dmctl.1714131665.67879.out cov.sequence_sharding_optimistic.dmctl.1714131665.67917.out cov.sequence_sharding_optimistic.dmctl.1714131665.67952.out cov.sequence_sharding_optimistic.dmctl.1714131665.67991.out cov.sequence_sharding_optimistic.dmctl.1714131666.68024.out cov.sequence_sharding_optimistic.dmctl.1714131666.68066.out cov.sequence_sharding_optimistic.dmctl.1714131666.68099.out cov.sequence_sharding_optimistic.dmctl.1714131666.68140.out cov.sequence_sharding_optimistic.dmctl.1714131666.68221.out cov.sequence_sharding_optimistic.dmctl.1714131666.68258.out cov.sequence_sharding_optimistic.dmctl.1714131666.68321.out cov.sequence_sharding_optimistic.dmctl.1714131667.68359.out cov.sequence_sharding_optimistic.dmctl.1714131667.68399.out cov.sequence_sharding_optimistic.dmctl.1714131667.68452.out cov.sequence_sharding_optimistic.dmctl.1714131667.68493.out cov.sequence_sharding_optimistic.master.out cov.sequence_sharding_optimistic.worker.8262.1714131654.out cov.sequence_sharding_optimistic.worker.8262.1714131663.out cov.sequence_sharding_optimistic.worker.8263.1714131655.out cov.sequence_sharding_removemeta.dmctl.1714131676.68963.out cov.sequence_sharding_removemeta.dmctl.1714131676.69005.out cov.sequence_sharding_removemeta.dmctl.1714131677.69048.out cov.sequence_sharding_removemeta.dmctl.1714131682.69164.out cov.sequence_sharding_removemeta.dmctl.1714131682.69205.out cov.sequence_sharding_removemeta.dmctl.1714131682.69262.out cov.sequence_sharding_removemeta.dmctl.1714131682.69317.out cov.sequence_sharding_removemeta.dmctl.1714131687.69378.out cov.sequence_sharding_removemeta.dmctl.1714131687.69459.out cov.sequence_sharding_removemeta.dmctl.1714131687.69504.out cov.sequence_sharding_removemeta.dmctl.1714131687.69552.out cov.sequence_sharding_removemeta.dmctl.1714131687.69597.out cov.sequence_sharding_removemeta.dmctl.1714131697.69641.out cov.sequence_sharding_removemeta.master.out cov.sequence_sharding_removemeta.worker.8262.1714131674.out cov.sequence_sharding_removemeta.worker.8263.1714131675.out cov.shardddl_optimistic.dmctl.1714131705.70000.out cov.shardddl_optimistic.dmctl.1714131706.70124.out cov.shardddl_optimistic.dmctl.1714131707.70240.out cov.shardddl_optimistic.dmctl.1714131709.70294.out cov.shardddl_optimistic.dmctl.1714131709.70471.out cov.shardddl_optimistic.dmctl.1714131710.70654.out cov.shardddl_optimistic.dmctl.1714131711.70698.out cov.shardddl_optimistic.dmctl.1714131711.70754.out cov.shardddl_optimistic.dmctl.1714131711.70807.out cov.shardddl_optimistic.dmctl.1714131712.70946.out cov.shardddl_optimistic.dmctl.1714131713.70984.out cov.shardddl_optimistic.dmctl.1714131713.71020.out cov.shardddl_optimistic.dmctl.1714131713.71060.out cov.shardddl_optimistic.dmctl.1714131713.71097.out cov.shardddl_optimistic.dmctl.1714131713.71135.out cov.shardddl_optimistic.dmctl.1714131713.71172.out cov.shardddl_optimistic.dmctl.1714131716.71253.out cov.shardddl_optimistic.dmctl.1714131716.71436.out cov.shardddl_optimistic.dmctl.1714131717.71481.out cov.shardddl_optimistic.dmctl.1714131760.72289.out cov.shardddl_optimistic.dmctl.1714131761.72446.out cov.shardddl_optimistic.dmctl.1714131762.72506.out cov.shardddl_optimistic.dmctl.1714131764.72640.out cov.shardddl_optimistic.dmctl.1714131766.72713.out cov.shardddl_optimistic.dmctl.1714131768.72817.out cov.shardddl_optimistic.dmctl.1714131768.72975.out cov.shardddl_optimistic.dmctl.1714131769.73026.out cov.shardddl_optimistic.dmctl.1714131772.73172.out cov.shardddl_optimistic.dmctl.1714131772.73214.out cov.shardddl_optimistic.dmctl.1714131773.73251.out cov.shardddl_optimistic.dmctl.1714131777.73335.out cov.shardddl_optimistic.dmctl.1714131777.73376.out cov.shardddl_optimistic.dmctl.1714131779.73437.out cov.shardddl_optimistic.dmctl.1714131779.73482.out cov.shardddl_optimistic.dmctl.1714131779.73584.out cov.shardddl_optimistic.dmctl.1714131779.73733.out cov.shardddl_optimistic.dmctl.1714131781.73780.out cov.shardddl_optimistic.dmctl.1714131785.73987.out cov.shardddl_optimistic.dmctl.1714131785.74024.out cov.shardddl_optimistic.master.out cov.shardddl_optimistic.worker.8262.1714131704.out cov.shardddl_optimistic.worker.8263.1714131705.out cov.shardddl_optimistic.worker.8263.1714131738.out cov.shardddl_optimistic.worker.8263.1714131759.out cov.slow_relay_writer.dmctl.1714131793.74500.out cov.slow_relay_writer.dmctl.1714131794.74543.out cov.slow_relay_writer.dmctl.1714131806.74722.out cov.slow_relay_writer.master.out cov.slow_relay_writer.worker.8262.1714131792.out cov.sql_mode.dmctl.1714131814.75160.out cov.sql_mode.dmctl.1714131815.75204.out cov.sql_mode.dmctl.1714131817.75264.out cov.sql_mode.master.out cov.sql_mode.worker.8262.1714131812.out cov.sql_mode.worker.8263.1714131813.out cov.sync_collation.dmctl.1714131829.75904.out cov.sync_collation.dmctl.1714131829.75946.out cov.sync_collation.dmctl.1714131830.76008.out cov.sync_collation.dmctl.1714131834.76161.out cov.sync_collation.dmctl.1714131834.76226.out cov.sync_collation.dmctl.1714131835.76275.out cov.sync_collation.dmctl.1714131836.76315.out cov.sync_collation.dmctl.1714131836.76373.out cov.sync_collation.dmctl.1714131837.76442.out cov.sync_collation.master.out cov.sync_collation.worker.8262.1714131827.out cov.sync_collation.worker.8263.1714131828.out cov.tracker_ignored_ddl.dmctl.1714131846.76803.out cov.tracker_ignored_ddl.dmctl.1714131846.76846.out cov.tracker_ignored_ddl.dmctl.1714131849.76947.out cov.tracker_ignored_ddl.dmctl.1714131849.76988.out cov.tracker_ignored_ddl.dmctl.1714131850.77029.out cov.tracker_ignored_ddl.dmctl.1714131850.77067.out cov.tracker_ignored_ddl.dmctl.1714131850.77113.out cov.tracker_ignored_ddl.dmctl.1714131850.77161.out cov.tracker_ignored_ddl.dmctl.1714131850.77198.out cov.tracker_ignored_ddl.dmctl.1714131850.77236.out cov.tracker_ignored_ddl.dmctl.1714131850.77365.out cov.tracker_ignored_ddl.dmctl.1714131852.77442.out cov.tracker_ignored_ddl.dmctl.1714131855.77533.out cov.tracker_ignored_ddl.dmctl.1714131855.77573.out cov.tracker_ignored_ddl.dmctl.1714131855.77612.out cov.tracker_ignored_ddl.dmctl.1714131855.77649.out cov.tracker_ignored_ddl.dmctl.1714131856.77697.out cov.tracker_ignored_ddl.dmctl.1714131856.77739.out cov.tracker_ignored_ddl.dmctl.1714131856.77779.out cov.tracker_ignored_ddl.dmctl.1714131856.77819.out cov.tracker_ignored_ddl.master.out cov.tracker_ignored_ddl.worker.8262.1714131845.out cov.validator_basic.dmctl.1714131863.78184.out cov.validator_basic.dmctl.1714131866.78315.out cov.validator_basic.dmctl.1714131867.78375.out cov.validator_basic.dmctl.1714131867.78418.out cov.validator_basic.dmctl.1714131878.78810.out cov.validator_basic.dmctl.1714131879.78923.out cov.validator_basic.dmctl.1714131880.78984.out cov.validator_basic.dmctl.1714131881.79023.out cov.validator_basic.dmctl.1714131891.79401.out cov.validator_basic.dmctl.1714131892.79528.out cov.validator_basic.dmctl.1714131894.79585.out cov.validator_basic.dmctl.1714131894.79626.out cov.validator_basic.dmctl.1714131904.80016.out cov.validator_basic.dmctl.1714131907.80137.out cov.validator_basic.dmctl.1714131908.80196.out cov.validator_basic.dmctl.1714131908.80237.out cov.validator_basic.dmctl.1714131917.80594.out cov.validator_basic.dmctl.1714131919.80707.out cov.validator_basic.dmctl.1714131920.80765.out cov.validator_basic.dmctl.1714131931.81146.out cov.validator_basic.dmctl.1714131932.81273.out cov.validator_basic.dmctl.1714131933.81329.out cov.validator_basic.dmctl.1714131944.81705.out cov.validator_basic.dmctl.1714131945.81822.out cov.validator_basic.dmctl.1714131946.81888.out cov.validator_basic.dmctl.1714131950.81980.out cov.validator_basic.dmctl.1714131962.82368.out cov.validator_basic.dmctl.1714131964.82488.out cov.validator_basic.dmctl.1714131965.82545.out cov.validator_basic.dmctl.1714131975.82878.out cov.validator_basic.dmctl.1714131976.82998.out cov.validator_basic.dmctl.1714131983.83067.out cov.validator_basic.dmctl.1714131983.83110.out cov.validator_basic.dmctl.1714131983.83146.out cov.validator_basic.dmctl.1714131993.83538.out cov.validator_basic.dmctl.1714131995.83660.out cov.validator_basic.dmctl.1714131997.83733.out cov.validator_basic.dmctl.1714132009.84121.out cov.validator_basic.dmctl.1714132010.84243.out cov.validator_basic.dmctl.1714132013.84320.out cov.validator_basic.dmctl.1714132026.84720.out cov.validator_basic.dmctl.1714132027.84856.out cov.validator_basic.dmctl.1714132028.84906.out cov.validator_basic.dmctl.1714132030.84966.out cov.validator_basic.dmctl.1714132030.85041.out cov.validator_basic.dmctl.1714132030.85079.out cov.validator_basic.dmctl.1714132030.85115.out cov.validator_basic.dmctl.1714132032.85176.out cov.validator_basic.dmctl.1714132045.85613.out cov.validator_basic.dmctl.1714132046.85734.out cov.validator_basic.dmctl.1714132047.85783.out cov.validator_basic.dmctl.1714132049.85839.out cov.validator_basic.dmctl.1714132049.85914.out cov.validator_basic.dmctl.1714132049.85950.out cov.validator_basic.dmctl.1714132050.85990.out cov.validator_basic.dmctl.1714132051.86055.out cov.validator_basic.dmctl.1714132065.86526.out cov.validator_basic.dmctl.1714132066.86646.out cov.validator_basic.dmctl.1714132068.86692.out cov.validator_basic.dmctl.1714132069.86750.out cov.validator_basic.dmctl.1714132069.86818.out cov.validator_basic.dmctl.1714132070.86859.out cov.validator_basic.dmctl.1714132070.86895.out cov.validator_basic.dmctl.1714132071.86954.out cov.validator_basic.dmctl.1714132084.87391.out cov.validator_basic.dmctl.1714132085.87514.out cov.validator_basic.dmctl.1714132087.87570.out cov.validator_basic.dmctl.1714132088.87625.out cov.validator_basic.dmctl.1714132089.87703.out cov.validator_basic.dmctl.1714132089.87740.out cov.validator_basic.dmctl.1714132089.87788.out cov.validator_basic.dmctl.1714132090.87854.out cov.validator_basic.dmctl.1714132103.88281.out cov.validator_basic.dmctl.1714132105.88412.out cov.validator_basic.dmctl.1714132106.88460.out cov.validator_basic.dmctl.1714132107.88515.out cov.validator_basic.dmctl.1714132108.88585.out cov.validator_basic.dmctl.1714132109.88639.out cov.validator_basic.dmctl.1714132109.88678.out cov.validator_basic.dmctl.1714132111.88734.out cov.validator_basic.dmctl.1714132120.89093.out cov.validator_basic.dmctl.1714132121.89212.out cov.validator_basic.dmctl.1714132122.89259.out cov.validator_basic.dmctl.1714132124.89314.out cov.validator_basic.dmctl.1714132124.89391.out cov.validator_basic.dmctl.1714132126.89457.out cov.validator_basic.dmctl.1714132126.89500.out cov.validator_basic.dmctl.1714132127.89549.out cov.validator_basic.dmctl.1714132136.89902.out cov.validator_basic.dmctl.1714132138.90020.out cov.validator_basic.dmctl.1714132139.90068.out cov.validator_basic.dmctl.1714132141.90127.out cov.validator_basic.dmctl.1714132141.90203.out cov.validator_basic.dmctl.1714132143.90283.out cov.validator_basic.dmctl.1714132145.90330.out cov.validator_basic.dmctl.1714132158.90745.out cov.validator_basic.dmctl.1714132159.90867.out cov.validator_basic.dmctl.1714132160.90916.out cov.validator_basic.dmctl.1714132162.90978.out cov.validator_basic.dmctl.1714132162.91049.out cov.validator_basic.dmctl.1714132164.91126.out cov.validator_basic.dmctl.1714132166.91179.out cov.validator_basic.dmctl.1714132179.91608.out cov.validator_basic.dmctl.1714132181.91730.out cov.validator_basic.dmctl.1714132182.91779.out cov.validator_basic.dmctl.1714132184.91835.out cov.validator_basic.dmctl.1714132184.91908.out cov.validator_basic.dmctl.1714132184.91951.out cov.validator_basic.dmctl.1714132184.91992.out cov.validator_basic.dmctl.1714132192.92323.out cov.validator_basic.dmctl.1714132193.92448.out cov.validator_basic.dmctl.1714132196.92528.out cov.validator_basic.dmctl.1714132209.92947.out cov.validator_basic.dmctl.1714132210.93064.out cov.validator_basic.dmctl.1714132213.93148.out cov.validator_basic.dmctl.1714132224.93543.out cov.validator_basic.dmctl.1714132227.93670.out cov.validator_basic.dmctl.1714132229.93761.out cov.validator_basic.dmctl.1714132241.94158.out cov.validator_basic.dmctl.1714132242.94278.out cov.validator_basic.dmctl.1714132245.94346.out cov.validator_basic.dmctl.1714132256.94738.out cov.validator_basic.dmctl.1714132257.94867.out cov.validator_basic.dmctl.1714132260.94942.out cov.validator_basic.dmctl.1714132260.94988.out cov.validator_basic.dmctl.1714132260.95028.out cov.validator_basic.dmctl.1714132269.95367.out cov.validator_basic.dmctl.1714132270.95491.out cov.validator_basic.dmctl.1714132273.95561.out cov.validator_basic.dmctl.1714132273.95608.out cov.validator_basic.dmctl.1714132273.95649.out cov.validator_basic.dmctl.1714132280.95973.out cov.validator_basic.dmctl.1714132282.96102.out cov.validator_basic.dmctl.1714132285.96179.out cov.validator_basic.dmctl.1714132285.96229.out cov.validator_basic.dmctl.1714132285.96266.out cov.validator_basic.dmctl.1714132292.96585.out cov.validator_basic.dmctl.1714132294.96716.out cov.validator_basic.dmctl.1714132297.96804.out cov.validator_basic.dmctl.1714132306.96956.out cov.validator_basic.dmctl.1714132306.96995.out cov.validator_basic.dmctl.1714132308.97073.out cov.validator_basic.dmctl.1714132309.97124.out cov.validator_basic.dmctl.1714132309.97167.out cov.validator_basic.dmctl.1714132310.97213.out cov.validator_basic.dmctl.1714132322.97586.out cov.validator_basic.dmctl.1714132323.97705.out cov.validator_basic.dmctl.1714132326.97783.out cov.validator_basic.dmctl.1714132332.97927.out cov.validator_basic.dmctl.1714132332.97967.out cov.validator_basic.dmctl.1714132334.98041.out cov.validator_basic.dmctl.1714132334.98087.out cov.validator_basic.dmctl.1714132346.98472.out cov.validator_basic.dmctl.1714132347.98596.out cov.validator_basic.dmctl.1714132354.98669.out cov.validator_basic.dmctl.1714132364.98918.out cov.validator_basic.dmctl.1714132366.98965.out cov.validator_basic.dmctl.1714132368.99014.out cov.validator_basic.dmctl.1714132368.99052.out cov.validator_basic.dmctl.1714132368.99090.out cov.validator_basic.dmctl.1714132370.99136.out cov.validator_basic.dmctl.1714132384.99428.out cov.validator_basic.dmctl.1714132384.99468.out cov.validator_basic.dmctl.1714132384.99507.out cov.validator_basic.dmctl.1714132386.99551.out cov.validator_basic.dmctl.1714132402.99879.out cov.validator_basic.dmctl.1714132403.99916.out cov.validator_basic.dmctl.1714132403.99957.out cov.validator_basic.dmctl.1714132404.100002.out cov.validator_basic.dmctl.1714132421.100321.out cov.validator_basic.dmctl.1714132421.100365.out cov.validator_basic.dmctl.1714132421.100402.out cov.validator_basic.dmctl.1714132423.100447.out cov.validator_basic.dmctl.1714132439.100759.out cov.validator_basic.dmctl.1714132439.100803.out cov.validator_basic.dmctl.1714132440.100844.out cov.validator_basic.dmctl.1714132464.101409.out cov.validator_basic.dmctl.1714132466.101532.out cov.validator_basic.dmctl.1714132467.101598.out cov.validator_basic.dmctl.1714132472.101689.out cov.validator_basic.dmctl.1714132483.102066.out cov.validator_basic.dmctl.1714132484.102189.out cov.validator_basic.dmctl.1714132486.102247.out cov.validator_basic.dmctl.1714132487.102300.out cov.validator_basic.dmctl.1714132490.102387.out cov.validator_basic.dmctl.1714132493.102431.out cov.validator_basic.dmctl.1714132494.102482.out cov.validator_basic.dmctl.1714132495.102567.out cov.validator_basic.dmctl.1714132497.102665.out cov.validator_basic.dmctl.1714132498.102713.out cov.validator_basic.dmctl.1714132499.102777.out cov.validator_basic.master.out cov.validator_basic.worker.8262.1714131862.out cov.validator_basic.worker.8262.1714131877.out cov.validator_basic.worker.8262.1714131890.out cov.validator_basic.worker.8262.1714131903.out cov.validator_basic.worker.8262.1714131916.out cov.validator_basic.worker.8262.1714131930.out cov.validator_basic.worker.8262.1714131943.out cov.validator_basic.worker.8262.1714131961.out cov.validator_basic.worker.8262.1714131974.out cov.validator_basic.worker.8262.1714131992.out cov.validator_basic.worker.8262.1714132008.out cov.validator_basic.worker.8262.1714132025.out cov.validator_basic.worker.8262.1714132044.out cov.validator_basic.worker.8262.1714132064.out cov.validator_basic.worker.8262.1714132083.out cov.validator_basic.worker.8262.1714132102.out cov.validator_basic.worker.8262.1714132119.out cov.validator_basic.worker.8262.1714132135.out cov.validator_basic.worker.8262.1714132157.out cov.validator_basic.worker.8262.1714132178.out cov.validator_basic.worker.8262.1714132191.out cov.validator_basic.worker.8262.1714132208.out cov.validator_basic.worker.8262.1714132223.out cov.validator_basic.worker.8262.1714132240.out cov.validator_basic.worker.8262.1714132255.out cov.validator_basic.worker.8262.1714132268.out cov.validator_basic.worker.8262.1714132279.out cov.validator_basic.worker.8262.1714132291.out cov.validator_basic.worker.8262.1714132321.out cov.validator_basic.worker.8262.1714132345.out cov.validator_basic.worker.8262.1714132463.out cov.validator_basic.worker.8262.1714132482.out cov.validator_basic.worker.8263.1714131865.out cov.validator_basic.worker.8263.1714131878.out cov.validator_basic.worker.8263.1714131891.out cov.validator_basic.worker.8263.1714131905.out cov.validator_basic.worker.8263.1714131918.out cov.validator_basic.worker.8263.1714131931.out cov.validator_basic.worker.8263.1714131944.out cov.validator_basic.worker.8263.1714131963.out cov.validator_basic.worker.8263.1714131975.out cov.validator_basic.worker.8263.1714131994.out cov.validator_basic.worker.8263.1714132009.out cov.validator_basic.worker.8263.1714132026.out cov.validator_basic.worker.8263.1714132045.out cov.validator_basic.worker.8263.1714132065.out cov.validator_basic.worker.8263.1714132084.out cov.validator_basic.worker.8263.1714132103.out cov.validator_basic.worker.8263.1714132120.out cov.validator_basic.worker.8263.1714132137.out cov.validator_basic.worker.8263.1714132158.out cov.validator_basic.worker.8263.1714132180.out cov.validator_basic.worker.8263.1714132192.out cov.validator_basic.worker.8263.1714132209.out cov.validator_basic.worker.8263.1714132226.out cov.validator_basic.worker.8263.1714132241.out cov.validator_basic.worker.8263.1714132256.out cov.validator_basic.worker.8263.1714132269.out cov.validator_basic.worker.8263.1714132281.out cov.validator_basic.worker.8263.1714132293.out cov.validator_basic.worker.8263.1714132322.out cov.validator_basic.worker.8263.1714132346.out cov.validator_basic.worker.8263.1714132465.out cov.validator_basic.worker.8263.1714132483.out cov.validator_checkpoint.dmctl.1714132508.103278.out cov.validator_checkpoint.dmctl.1714132509.103322.out cov.validator_checkpoint.dmctl.1714132510.103369.out cov.validator_checkpoint.dmctl.1714132511.103421.out cov.validator_checkpoint.dmctl.1714132511.103495.out cov.validator_checkpoint.dmctl.1714132520.103862.out cov.validator_checkpoint.dmctl.1714132522.103966.out cov.validator_checkpoint.dmctl.1714132535.104453.out cov.validator_checkpoint.dmctl.1714132535.104501.out cov.validator_checkpoint.dmctl.1714132536.104540.out cov.validator_checkpoint.dmctl.1714132538.104606.out cov.validator_checkpoint.dmctl.1714132538.104710.out cov.validator_checkpoint.dmctl.1714132541.104849.out cov.validator_checkpoint.dmctl.1714132541.104893.out cov.validator_checkpoint.dmctl.1714132542.104969.out cov.validator_checkpoint.dmctl.1714132545.105116.out cov.validator_checkpoint.master.out cov.validator_checkpoint.worker.8262.1714132506.out cov.validator_checkpoint.worker.8262.1714132516.out cov.validator_checkpoint.worker.8262.1714132533.out cov.validator_checkpoint.worker.8263.1714132507.out cov.validator_checkpoint.worker.8263.1714132519.out cov.validator_checkpoint.worker.8263.1714132534.out dm_syncer downstream downstream_diff_index downstream_more_column drop_column_with_index duplicate_event expression_filter extend_column fake_rotate_event foreign_key full_mode gbk goroutines gtid ha_cases http_proxies incompatible_ddl_changes lightning_load_task lightning_mode metrics new_collation_off only_dml openapi s3_dumpling_lightning s3_dumpling_lightning.downstream sequence_sharding_optimistic sequence_sharding_removemeta shardddl_optimistic slow_relay_writer sql_mode sql_res.adjust_gtid.txt sql_res.async_checkpoint_flush.txt sql_res.binlog_parse.txt sql_res.case_sensitive.txt sql_res.check_task.txt sql_res.checkpoint_transaction.txt sql_res.dm_syncer.txt sql_res.downstream_diff_index.txt sql_res.downstream_more_column.txt sql_res.drop_column_with_index.txt sql_res.duplicate_event.txt sql_res.expression_filter.txt sql_res.extend_column.txt sql_res.fake_rotate_event.txt sql_res.foreign_key.txt sql_res.full_mode.txt sql_res.gbk.txt sql_res.gtid.txt sql_res.ha_cases.txt sql_res.http_proxies.txt sql_res.incompatible_ddl_changes.txt sql_res.lightning_load_task.txt sql_res.lightning_mode.txt sql_res.metrics.txt sql_res.new_collation_off.txt sql_res.only_dml.txt sql_res.openapi.txt sql_res.s3_dumpling_lightning.txt sql_res.sequence_sharding_optimistic.txt sql_res.sequence_sharding_removemeta.txt sql_res.shardddl_optimistic.txt sql_res.slow_relay_writer.txt sql_res.sql_mode.txt sql_res.sync_collation.txt sql_res.tracker_ignored_ddl.txt sql_res.validator_basic.txt sql_res.validator_checkpoint.txt sync_collation tidb.toml tracker_ignored_ddl validator_basic validator_checkpoint ++ find /tmp/dm_test/ -type f -name '*.log' + tar -cvzf log-G11.tar.gz /tmp/dm_test/gtid/master/log/dm-master.log /tmp/dm_test/gtid/master/log/stdout.log /tmp/dm_test/gtid/dmctl.1714130888.log /tmp/dm_test/gtid/dmctl.1714130892.log /tmp/dm_test/gtid/worker2/log/dm-worker.log /tmp/dm_test/gtid/worker2/log/stdout.log /tmp/dm_test/gtid/dmctl.1714130890.log /tmp/dm_test/gtid/dmctl.1714130899.log /tmp/dm_test/gtid/dmctl.1714130897.log /tmp/dm_test/gtid/sync_diff_stdout.log /tmp/dm_test/gtid/worker1/log/dm-worker.log /tmp/dm_test/gtid/worker1/log/stdout.log /tmp/dm_test/gtid/dmctl.1714130894.log /tmp/dm_test/gtid/dmctl.1714130889.log /tmp/dm_test/gtid/dmctl.1714130893.log /tmp/dm_test/sequence_sharding_removemeta/master/log/dm-master.log /tmp/dm_test/sequence_sharding_removemeta/master/log/stdout.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131677.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131697.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131682.log /tmp/dm_test/sequence_sharding_removemeta/worker2/log/dm-worker.log /tmp/dm_test/sequence_sharding_removemeta/worker2/log/stdout.log /tmp/dm_test/sequence_sharding_removemeta/sync_diff_stdout.log /tmp/dm_test/sequence_sharding_removemeta/worker1/log/dm-worker.log /tmp/dm_test/sequence_sharding_removemeta/worker1/log/stdout.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131676.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131687.log /tmp/dm_test/lightning_mode/master/log/dm-master.log /tmp/dm_test/lightning_mode/master/log/stdout.log /tmp/dm_test/lightning_mode/dmctl.1714131219.log /tmp/dm_test/lightning_mode/dmctl.1714131225.log /tmp/dm_test/lightning_mode/dmctl.1714131188.log /tmp/dm_test/lightning_mode/tikv.log /tmp/dm_test/lightning_mode/dmctl.1714131212.log /tmp/dm_test/lightning_mode/pd/region-meta/000001.log /tmp/dm_test/lightning_mode/pd/hot-region/000001.log /tmp/dm_test/lightning_mode/dmctl.1714131226.log /tmp/dm_test/lightning_mode/tidb.log /tmp/dm_test/lightning_mode/worker2/log/dm-worker.log /tmp/dm_test/lightning_mode/worker2/log/stdout.log /tmp/dm_test/lightning_mode/dmctl.1714131203.log /tmp/dm_test/lightning_mode/dmctl.1714131195.log /tmp/dm_test/lightning_mode/pd.log /tmp/dm_test/lightning_mode/sync_diff_stdout.log /tmp/dm_test/lightning_mode/tikv/db/000005.log /tmp/dm_test/lightning_mode/worker1/log/dm-worker.log /tmp/dm_test/lightning_mode/worker1/log/stdout.log /tmp/dm_test/lightning_mode/dmctl.1714131189.log /tmp/dm_test/lightning_mode/dmctl.1714131197.log /tmp/dm_test/lightning_mode/dmctl.1714131190.log /tmp/dm_test/lightning_mode/dmctl.1714131210.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/region-meta/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/hot-region/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/tidb.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv/db/000005.log /tmp/dm_test/downstream_more_column/master/log/dm-master.log /tmp/dm_test/downstream_more_column/master/log/stdout.log /tmp/dm_test/downstream_more_column/dmctl.1714130456.log /tmp/dm_test/downstream_more_column/dmctl.1714130453.log /tmp/dm_test/downstream_more_column/dmctl.1714130452.log /tmp/dm_test/downstream_more_column/worker1/log/dm-worker.log /tmp/dm_test/downstream_more_column/worker1/log/stdout.log /tmp/dm_test/downstream_more_column/dmctl.1714130458.log /tmp/dm_test/lightning_load_task/master/log/dm-master.log /tmp/dm_test/lightning_load_task/master/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131130.log /tmp/dm_test/lightning_load_task/dmctl.1714131120.log /tmp/dm_test/lightning_load_task/dmctl.1714131134.log /tmp/dm_test/lightning_load_task/dmctl.1714131115.log /tmp/dm_test/lightning_load_task/dmctl.1714131124.log /tmp/dm_test/lightning_load_task/worker2/log/dm-worker.log /tmp/dm_test/lightning_load_task/worker2/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131117.log /tmp/dm_test/lightning_load_task/dmctl.1714131128.log /tmp/dm_test/lightning_load_task/dmctl.1714131148.log /tmp/dm_test/lightning_load_task/sync_diff_stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131135.log /tmp/dm_test/lightning_load_task/worker1/log/dm-worker.log /tmp/dm_test/lightning_load_task/worker1/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131137.log /tmp/dm_test/lightning_load_task/dmctl.1714131150.log /tmp/dm_test/lightning_load_task/dmctl.1714131125.log /tmp/dm_test/lightning_load_task/dmctl.1714131129.log /tmp/dm_test/lightning_load_task/dmctl.1714131121.log /tmp/dm_test/lightning_load_task/dmctl.1714131119.log /tmp/dm_test/lightning_load_task/dmctl.1714131149.log /tmp/dm_test/lightning_load_task/worker3/log/dm-worker.log /tmp/dm_test/lightning_load_task/worker3/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131132.log /tmp/dm_test/s3_dumpling_lightning/master/log/dm-master.log /tmp/dm_test/s3_dumpling_lightning/master/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714131637.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714131638.log /tmp/dm_test/sync_collation/master/log/dm-master.log /tmp/dm_test/sync_collation/master/log/stdout.log /tmp/dm_test/sync_collation/dmctl.1714131835.log /tmp/dm_test/sync_collation/worker2/log/dm-worker.log /tmp/dm_test/sync_collation/worker2/log/stdout.log /tmp/dm_test/sync_collation/dmctl.1714131830.log /tmp/dm_test/sync_collation/dmctl.1714131837.log /tmp/dm_test/sync_collation/dmctl.1714131829.log /tmp/dm_test/sync_collation/worker1/log/dm-worker.log /tmp/dm_test/sync_collation/worker1/log/stdout.log /tmp/dm_test/sync_collation/dmctl.1714131834.log /tmp/dm_test/sync_collation/dmctl.1714131836.log /tmp/dm_test/adjust_gtid/master/log/dm-master.log /tmp/dm_test/adjust_gtid/master/log/stdout.log /tmp/dm_test/adjust_gtid/worker2/log/dm-worker.log /tmp/dm_test/adjust_gtid/worker2/log/stdout.log /tmp/dm_test/adjust_gtid/dmctl.1714130193.log /tmp/dm_test/adjust_gtid/dmctl.1714130203.log /tmp/dm_test/adjust_gtid/dmctl.1714130192.log /tmp/dm_test/adjust_gtid/dmctl.1714130205.log /tmp/dm_test/adjust_gtid/sync_diff_stdout.log /tmp/dm_test/adjust_gtid/worker1/log/dm-worker.log /tmp/dm_test/adjust_gtid/worker1/log/stdout.log /tmp/dm_test/adjust_gtid/dmctl.1714130189.log /tmp/dm_test/adjust_gtid/dmctl.1714130191.log /tmp/dm_test/duplicate_event/dmctl.1714130643.log /tmp/dm_test/duplicate_event/master/log/dm-master.log /tmp/dm_test/duplicate_event/master/log/stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130642.log /tmp/dm_test/duplicate_event/worker2/log/dm-worker.log /tmp/dm_test/duplicate_event/worker2/log/stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130575.log /tmp/dm_test/duplicate_event/sync_diff_stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130641.log /tmp/dm_test/duplicate_event/worker1/log/dm-worker.log /tmp/dm_test/duplicate_event/worker1/log/stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130640.log /tmp/dm_test/downstream_diff_index/master/log/dm-master.log /tmp/dm_test/downstream_diff_index/master/log/stdout.log /tmp/dm_test/downstream_diff_index/dmctl.1714130439.log /tmp/dm_test/downstream_diff_index/worker2/log/dm-worker.log /tmp/dm_test/downstream_diff_index/worker2/log/stdout.log /tmp/dm_test/downstream_diff_index/dmctl.1714130441.log /tmp/dm_test/downstream_diff_index/worker1/log/dm-worker.log /tmp/dm_test/downstream_diff_index/worker1/log/stdout.log /tmp/dm_test/downstream_diff_index/dmctl.1714130440.log /tmp/dm_test/binlog_parse/master/log/dm-master.log /tmp/dm_test/binlog_parse/master/log/stdout.log /tmp/dm_test/binlog_parse/dmctl.1714130258.log /tmp/dm_test/binlog_parse/dmctl.1714130256.log /tmp/dm_test/binlog_parse/sync_diff_stdout.log /tmp/dm_test/binlog_parse/dmctl.1714130259.log /tmp/dm_test/binlog_parse/dmctl.1714130257.log /tmp/dm_test/binlog_parse/worker1/log/dm-worker.log /tmp/dm_test/binlog_parse/worker1/log/stdout.log /tmp/dm_test/binlog_parse/dmctl.1714130260.log /tmp/dm_test/goroutines/stack/log/master-8261.log /tmp/dm_test/goroutines/stack/log/master-8361.log /tmp/dm_test/goroutines/stack/log/worker-8262.log /tmp/dm_test/goroutines/stack/log/master-8661.log /tmp/dm_test/goroutines/stack/log/worker-18263.log /tmp/dm_test/goroutines/stack/log/master-8461.log /tmp/dm_test/goroutines/stack/log/worker-18262.log /tmp/dm_test/goroutines/stack/log/master-8561.log /tmp/dm_test/goroutines/stack/log/worker-8264.log /tmp/dm_test/goroutines/stack/log/worker-8263.log /tmp/dm_test/goroutines/stack/log/master-8761.log /tmp/dm_test/fake_rotate_event/master/log/dm-master.log /tmp/dm_test/fake_rotate_event/master/log/stdout.log /tmp/dm_test/fake_rotate_event/dmctl.1714130705.log /tmp/dm_test/fake_rotate_event/sync_diff_stdout.log /tmp/dm_test/fake_rotate_event/worker1/log/dm-worker.log /tmp/dm_test/fake_rotate_event/worker1/log/stdout.log /tmp/dm_test/fake_rotate_event/dmctl.1714130710.log /tmp/dm_test/fake_rotate_event/dmctl.1714130706.log /tmp/dm_test/sql_mode/master/log/dm-master.log /tmp/dm_test/sql_mode/master/log/stdout.log /tmp/dm_test/sql_mode/dmctl.1714131815.log /tmp/dm_test/sql_mode/worker2/log/dm-worker.log /tmp/dm_test/sql_mode/worker2/log/stdout.log /tmp/dm_test/sql_mode/dmctl.1714131814.log /tmp/dm_test/sql_mode/sync_diff_stdout.log /tmp/dm_test/sql_mode/worker1/log/dm-worker.log /tmp/dm_test/sql_mode/worker1/log/stdout.log /tmp/dm_test/sql_mode/dmctl.1714131817.log /tmp/dm_test/ha_cases/master2/log/dm-master.log /tmp/dm_test/ha_cases/master2/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131069.log /tmp/dm_test/ha_cases/worker2/log/dm-worker.log /tmp/dm_test/ha_cases/worker2/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131064.log /tmp/dm_test/ha_cases/dmctl.1714131067.log /tmp/dm_test/ha_cases/sync_diff_stdout.log /tmp/dm_test/ha_cases/worker1/log/dm-worker.log /tmp/dm_test/ha_cases/worker1/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131072.log /tmp/dm_test/ha_cases/master1/log/dm-master.log /tmp/dm_test/ha_cases/master1/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131070.log /tmp/dm_test/ha_cases/dmctl.1714131063.log /tmp/dm_test/ha_cases/dmctl.1714131068.log /tmp/dm_test/ha_cases/dmctl.1714131073.log /tmp/dm_test/ha_cases/master3/log/dm-master.log /tmp/dm_test/ha_cases/master3/log/stdout.log /tmp/dm_test/new_collation_off/master/log/dm-master.log /tmp/dm_test/new_collation_off/master/log/stdout.log /tmp/dm_test/new_collation_off/sync_diff_stdout.log /tmp/dm_test/new_collation_off/worker1/log/dm-worker.log /tmp/dm_test/new_collation_off/worker1/log/stdout.log /tmp/dm_test/new_collation_off/dmctl.1714131292.log /tmp/dm_test/sequence_sharding_optimistic/master/log/dm-master.log /tmp/dm_test/sequence_sharding_optimistic/master/log/stdout.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131666.log /tmp/dm_test/sequence_sharding_optimistic/remove_schema.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131656.log /tmp/dm_test/sequence_sharding_optimistic/set_schema.log /tmp/dm_test/sequence_sharding_optimistic/worker2/log/dm-worker.log /tmp/dm_test/sequence_sharding_optimistic/worker2/log/stdout.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131661.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131657.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131667.log /tmp/dm_test/sequence_sharding_optimistic/sync_diff_stdout.log /tmp/dm_test/sequence_sharding_optimistic/worker1/log/dm-worker.log /tmp/dm_test/sequence_sharding_optimistic/worker1/log/stdout.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131665.log /tmp/dm_test/sequence_sharding_optimistic/get_schema.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131664.log /tmp/dm_test/gbk/master/log/dm-master.log /tmp/dm_test/gbk/master/log/stdout.log /tmp/dm_test/gbk/dmctl.1714130833.log /tmp/dm_test/gbk/worker2/log/dm-worker.log /tmp/dm_test/gbk/worker2/log/stdout.log /tmp/dm_test/gbk/dmctl.1714130865.log /tmp/dm_test/gbk/dmctl.1714130836.log /tmp/dm_test/gbk/sync_diff_stdout.log /tmp/dm_test/gbk/worker1/log/dm-worker.log /tmp/dm_test/gbk/worker1/log/stdout.log /tmp/dm_test/gbk/dmctl.1714130796.log /tmp/dm_test/gbk/dmctl.1714130795.log /tmp/dm_test/dm_syncer/master/log/dm-master.log /tmp/dm_test/dm_syncer/master/log/stdout.log /tmp/dm_test/dm_syncer/worker2/log/dm-worker.log /tmp/dm_test/dm_syncer/worker2/log/stdout.log /tmp/dm_test/dm_syncer/dmctl.1714130422.log /tmp/dm_test/dm_syncer/syncer1/log/stdout.log /tmp/dm_test/dm_syncer/syncer1/log/dm-syncer.log /tmp/dm_test/dm_syncer/dmctl.1714130419.log /tmp/dm_test/dm_syncer/sync_diff_stdout.log /tmp/dm_test/dm_syncer/syncer2/log/stdout.log /tmp/dm_test/dm_syncer/syncer2/log/dm-syncer.log /tmp/dm_test/dm_syncer/worker1/log/dm-worker.log /tmp/dm_test/dm_syncer/worker1/log/stdout.log /tmp/dm_test/dm_syncer/dmctl.1714130420.log /tmp/dm_test/extend_column/master/log/dm-master.log /tmp/dm_test/extend_column/master/log/stdout.log /tmp/dm_test/extend_column/worker2/log/dm-worker.log /tmp/dm_test/extend_column/worker2/log/stdout.log /tmp/dm_test/extend_column/dmctl.1714130692.log /tmp/dm_test/extend_column/worker1/log/dm-worker.log /tmp/dm_test/extend_column/worker1/log/stdout.log /tmp/dm_test/extend_column/dmctl.1714130694.log /tmp/dm_test/openapi/master2/log/dm-master.log /tmp/dm_test/openapi/master2/log/stdout.log /tmp/dm_test/openapi/worker2/log/dm-worker.log /tmp/dm_test/openapi/worker2/log/stdout.log /tmp/dm_test/openapi/worker1/log/dm-worker.log /tmp/dm_test/openapi/worker1/log/stdout.log /tmp/dm_test/openapi/master1/log/dm-master.log /tmp/dm_test/openapi/master1/log/stdout.log /tmp/dm_test/shardddl_optimistic/master/log/dm-master.log /tmp/dm_test/shardddl_optimistic/master/log/stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131760.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131707.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131713.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131706.log /tmp/dm_test/shardddl_optimistic/worker2/log/dm-worker.log /tmp/dm_test/shardddl_optimistic/worker2/log/stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131711.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131717.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131761.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131705.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131772.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131777.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131781.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131709.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131712.log /tmp/dm_test/shardddl_optimistic/sync_diff_stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131779.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131764.log /tmp/dm_test/shardddl_optimistic/worker1/log/dm-worker.log /tmp/dm_test/shardddl_optimistic/worker1/log/stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131762.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131766.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131710.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131773.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131768.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131785.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131716.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131769.log /tmp/dm_test/foreign_key/master/log/dm-master.log /tmp/dm_test/foreign_key/master/log/stdout.log /tmp/dm_test/foreign_key/dmctl.1714130720.log /tmp/dm_test/foreign_key/sync_diff_stdout.log /tmp/dm_test/foreign_key/worker1/log/dm-worker.log /tmp/dm_test/foreign_key/worker1/log/stdout.log /tmp/dm_test/foreign_key/dmctl.1714130718.log /tmp/dm_test/drop_column_with_index/master/log/dm-master.log /tmp/dm_test/drop_column_with_index/master/log/stdout.log /tmp/dm_test/drop_column_with_index/dmctl.1714130469.log /tmp/dm_test/drop_column_with_index/dmctl.1714130467.log /tmp/dm_test/drop_column_with_index/dmctl.1714130468.log /tmp/dm_test/drop_column_with_index/dmctl.1714130470.log /tmp/dm_test/drop_column_with_index/sync_diff_stdout.log /tmp/dm_test/drop_column_with_index/worker1/log/dm-worker.log /tmp/dm_test/drop_column_with_index/worker1/log/stdout.log /tmp/dm_test/drop_column_with_index/dmctl.1714130475.log /tmp/dm_test/expression_filter/master/log/dm-master.log /tmp/dm_test/expression_filter/master/log/stdout.log /tmp/dm_test/expression_filter/dmctl.1714130664.log /tmp/dm_test/expression_filter/worker1/log/dm-worker.log /tmp/dm_test/expression_filter/worker1/log/stdout.log /tmp/dm_test/expression_filter/dmctl.1714130670.log /tmp/dm_test/expression_filter/dmctl.1714130663.log /tmp/dm_test/slow_relay_writer/dmctl.1714131794.log /tmp/dm_test/slow_relay_writer/master/log/dm-master.log /tmp/dm_test/slow_relay_writer/master/log/stdout.log /tmp/dm_test/slow_relay_writer/dmctl.1714131806.log /tmp/dm_test/slow_relay_writer/sync_diff_stdout.log /tmp/dm_test/slow_relay_writer/worker1/log/dm-worker.log /tmp/dm_test/slow_relay_writer/worker1/log/stdout.log /tmp/dm_test/slow_relay_writer/dmctl.1714131793.log /tmp/dm_test/tracker_ignored_ddl/master/log/dm-master.log /tmp/dm_test/tracker_ignored_ddl/master/log/stdout.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131846.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131852.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131849.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131850.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131855.log /tmp/dm_test/tracker_ignored_ddl/sync_diff_stdout.log /tmp/dm_test/tracker_ignored_ddl/worker1/log/dm-worker.log /tmp/dm_test/tracker_ignored_ddl/worker1/log/stdout.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131856.log /tmp/dm_test/metrics/master/log/dm-master.log /tmp/dm_test/metrics/master/log/stdout.log /tmp/dm_test/metrics/dmctl.1714131255.log /tmp/dm_test/metrics/worker2/log/dm-worker.log /tmp/dm_test/metrics/worker2/log/stdout.log /tmp/dm_test/metrics/dmctl.1714131256.log /tmp/dm_test/metrics/dmctl.1714131254.log /tmp/dm_test/metrics/dmctl.1714131280.log /tmp/dm_test/metrics/dmctl.1714131252.log /tmp/dm_test/metrics/dmctl.1714131277.log /tmp/dm_test/metrics/sync_diff_stdout.log /tmp/dm_test/metrics/dmctl.1714131260.log /tmp/dm_test/metrics/worker1/log/dm-worker.log /tmp/dm_test/metrics/worker1/log/stdout.log /tmp/dm_test/checkpoint_transaction/master/log/dm-master.log /tmp/dm_test/checkpoint_transaction/master/log/stdout.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130364.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130333.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130334.log /tmp/dm_test/checkpoint_transaction/sync_diff_stdout.log /tmp/dm_test/checkpoint_transaction/worker1/log/dm-worker.log /tmp/dm_test/checkpoint_transaction/worker1/log/stdout.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130379.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130382.log /tmp/dm_test/incompatible_ddl_changes/master/log/dm-master.log /tmp/dm_test/incompatible_ddl_changes/master/log/stdout.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132563.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132556.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132558.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132562.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132561.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132560.log /tmp/dm_test/incompatible_ddl_changes/worker1/log/dm-worker.log /tmp/dm_test/incompatible_ddl_changes/worker1/log/stdout.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132557.log /tmp/dm_test/downstream/tidb/log/tidb.log /tmp/dm_test/http_proxies/master/log/dm-master.log /tmp/dm_test/http_proxies/master/log/stdout.log /tmp/dm_test/http_proxies/dmctl.1714131095.log /tmp/dm_test/http_proxies/dmctl.1714131084.log /tmp/dm_test/http_proxies/worker1/log/dm-worker.log /tmp/dm_test/http_proxies/worker1/log/stdout.log /tmp/dm_test/http_proxies/dmctl.1714131106.log /tmp/dm_test/only_dml/dmctl.1714131341.log /tmp/dm_test/only_dml/dmctl.1714131332.log /tmp/dm_test/only_dml/dmctl.1714131337.log /tmp/dm_test/only_dml/master/log/dm-master.log /tmp/dm_test/only_dml/master/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714131302.log /tmp/dm_test/only_dml/dmctl.1714131325.log /tmp/dm_test/only_dml/dmctl.1714131342.log /tmp/dm_test/only_dml/dmctl.1714131328.log /tmp/dm_test/only_dml/dmctl.1714131308.log /tmp/dm_test/only_dml/dmctl.1714131338.log /tmp/dm_test/only_dml/dmctl.1714131326.log /tmp/dm_test/only_dml/dmctl.1714131329.log /tmp/dm_test/only_dml/dmctl.1714131305.log /tmp/dm_test/only_dml/dmctl.1714131314.log /tmp/dm_test/only_dml/worker2/log/dm-worker.log /tmp/dm_test/only_dml/worker2/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714131339.log /tmp/dm_test/only_dml/dmctl.1714131333.log /tmp/dm_test/only_dml/dmctl.1714131301.log /tmp/dm_test/only_dml/dmctl.1714131315.log /tmp/dm_test/only_dml/dmctl.1714131307.log /tmp/dm_test/only_dml/dmctl.1714131321.log /tmp/dm_test/only_dml/dmctl.1714131311.log /tmp/dm_test/only_dml/dmctl.1714131345.log /tmp/dm_test/only_dml/dmctl.1714131336.log /tmp/dm_test/only_dml/dmctl.1714131343.log /tmp/dm_test/only_dml/dmctl.1714131340.log /tmp/dm_test/only_dml/dmctl.1714131334.log /tmp/dm_test/only_dml/sync_diff_stdout.log /tmp/dm_test/only_dml/dmctl.1714131331.log /tmp/dm_test/only_dml/dmctl.1714131324.log /tmp/dm_test/only_dml/dmctl.1714131306.log /tmp/dm_test/only_dml/dmctl.1714131309.log /tmp/dm_test/only_dml/worker1/log/dm-worker.log /tmp/dm_test/only_dml/worker1/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714131312.log /tmp/dm_test/only_dml/dmctl.1714131330.log /tmp/dm_test/only_dml/dmctl.1714131313.log /tmp/dm_test/only_dml/dmctl.1714131304.log /tmp/dm_test/only_dml/dmctl.1714131323.log /tmp/dm_test/only_dml/dmctl.1714131322.log /tmp/dm_test/only_dml/dmctl.1714131320.log /tmp/dm_test/only_dml/dmctl.1714131319.log /tmp/dm_test/only_dml/dmctl.1714131317.log /tmp/dm_test/only_dml/dmctl.1714131316.log /tmp/dm_test/full_mode/master/log/dm-master.log /tmp/dm_test/full_mode/master/log/stdout.log /tmp/dm_test/full_mode/dmctl.1714130782.log /tmp/dm_test/full_mode/dmctl.1714130783.log /tmp/dm_test/full_mode/worker2/log/dm-worker.log /tmp/dm_test/full_mode/worker2/log/stdout.log /tmp/dm_test/full_mode/dmctl.1714130785.log /tmp/dm_test/full_mode/sync_diff_stdout.log /tmp/dm_test/full_mode/worker1/log/dm-worker.log /tmp/dm_test/full_mode/worker1/log/stdout.log /tmp/dm_test/full_mode/dmctl.1714130784.log /tmp/dm_test/async_checkpoint_flush/master/log/dm-master.log /tmp/dm_test/async_checkpoint_flush/master/log/stdout.log /tmp/dm_test/async_checkpoint_flush/dmctl.1714130217.log /tmp/dm_test/async_checkpoint_flush/sync_diff_stdout.log /tmp/dm_test/async_checkpoint_flush/worker1/log/dm-worker.log /tmp/dm_test/async_checkpoint_flush/worker1/log/stdout.log tar: Removing leading `/' from member names /tmp/dm_test/gtid/master/log/dm-master.log /tmp/dm_test/gtid/master/log/stdout.log /tmp/dm_test/gtid/dmctl.1714130888.log /tmp/dm_test/gtid/dmctl.1714130892.log /tmp/dm_test/gtid/worker2/log/dm-worker.log /tmp/dm_test/gtid/worker2/log/stdout.log /tmp/dm_test/gtid/dmctl.1714130890.log /tmp/dm_test/gtid/dmctl.1714130899.log /tmp/dm_test/gtid/dmctl.1714130897.log /tmp/dm_test/gtid/sync_diff_stdout.log /tmp/dm_test/gtid/worker1/log/dm-worker.log /tmp/dm_test/gtid/worker1/log/stdout.log /tmp/dm_test/gtid/dmctl.1714130894.log /tmp/dm_test/gtid/dmctl.1714130889.log /tmp/dm_test/gtid/dmctl.1714130893.log /tmp/dm_test/sequence_sharding_removemeta/master/log/dm-master.log /tmp/dm_test/sequence_sharding_removemeta/master/log/stdout.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131677.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131697.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131682.log /tmp/dm_test/sequence_sharding_removemeta/worker2/log/dm-worker.log /tmp/dm_test/sequence_sharding_removemeta/worker2/log/stdout.log /tmp/dm_test/sequence_sharding_removemeta/sync_diff_stdout.log /tmp/dm_test/sequence_sharding_removemeta/worker1/log/dm-worker.log /tmp/dm_test/sequence_sharding_removemeta/worker1/log/stdout.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131676.log /tmp/dm_test/sequence_sharding_removemeta/dmctl.1714131687.log /tmp/dm_test/lightning_mode/master/log/dm-master.log /tmp/dm_test/lightning_mode/master/log/stdout.log /tmp/dm_test/lightning_mode/dmctl.1714131219.log /tmp/dm_test/lightning_mode/dmctl.1714131225.log /tmp/dm_test/lightning_mode/dmctl.1714131188.log /tmp/dm_test/lightning_mode/tikv.log /tmp/dm_test/lightning_mode/dmctl.1714131212.log /tmp/dm_test/lightning_mode/pd/region-meta/000001.log /tmp/dm_test/lightning_mode/pd/hot-region/000001.log /tmp/dm_test/lightning_mode/dmctl.1714131226.log /tmp/dm_test/lightning_mode/tidb.log /tmp/dm_test/lightning_mode/worker2/log/dm-worker.log /tmp/dm_test/lightning_mode/worker2/log/stdout.log /tmp/dm_test/lightning_mode/dmctl.1714131203.log /tmp/dm_test/lightning_mode/dmctl.1714131195.log /tmp/dm_test/lightning_mode/pd.log /tmp/dm_test/lightning_mode/sync_diff_stdout.log /tmp/dm_test/lightning_mode/tikv/db/000005.log /tmp/dm_test/lightning_mode/worker1/log/dm-worker.log /tmp/dm_test/lightning_mode/worker1/log/stdout.log /tmp/dm_test/lightning_mode/dmctl.1714131189.log /tmp/dm_test/lightning_mode/dmctl.1714131197.log /tmp/dm_test/lightning_mode/dmctl.1714131190.log /tmp/dm_test/lightning_mode/dmctl.1714131210.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/region-meta/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/hot-region/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/tidb.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv/db/000005.log /tmp/dm_test/downstream_more_column/master/log/dm-master.log /tmp/dm_test/downstream_more_column/master/log/stdout.log /tmp/dm_test/downstream_more_column/dmctl.1714130456.log /tmp/dm_test/downstream_more_column/dmctl.1714130453.log /tmp/dm_test/downstream_more_column/dmctl.1714130452.log /tmp/dm_test/downstream_more_column/worker1/log/dm-worker.log /tmp/dm_test/downstream_more_column/worker1/log/stdout.log /tmp/dm_test/downstream_more_column/dmctl.1714130458.log /tmp/dm_test/lightning_load_task/master/log/dm-master.log /tmp/dm_test/lightning_load_task/master/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131130.log /tmp/dm_test/lightning_load_task/dmctl.1714131120.log /tmp/dm_test/lightning_load_task/dmctl.1714131134.log /tmp/dm_test/lightning_load_task/dmctl.1714131115.log /tmp/dm_test/lightning_load_task/dmctl.1714131124.log /tmp/dm_test/lightning_load_task/worker2/log/dm-worker.log /tmp/dm_test/lightning_load_task/worker2/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131117.log /tmp/dm_test/lightning_load_task/dmctl.1714131128.log /tmp/dm_test/lightning_load_task/dmctl.1714131148.log /tmp/dm_test/lightning_load_task/sync_diff_stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131135.log /tmp/dm_test/lightning_load_task/worker1/log/dm-worker.log /tmp/dm_test/lightning_load_task/worker1/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131137.log /tmp/dm_test/lightning_load_task/dmctl.1714131150.log /tmp/dm_test/lightning_load_task/dmctl.1714131125.log /tmp/dm_test/lightning_load_task/dmctl.1714131129.log /tmp/dm_test/lightning_load_task/dmctl.1714131121.log /tmp/dm_test/lightning_load_task/dmctl.1714131119.log /tmp/dm_test/lightning_load_task/dmctl.1714131149.log /tmp/dm_test/lightning_load_task/worker3/log/dm-worker.log /tmp/dm_test/lightning_load_task/worker3/log/stdout.log /tmp/dm_test/lightning_load_task/dmctl.1714131132.log /tmp/dm_test/s3_dumpling_lightning/master/log/dm-master.log /tmp/dm_test/s3_dumpling_lightning/master/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714131637.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714131638.log /tmp/dm_test/sync_collation/master/log/dm-master.log /tmp/dm_test/sync_collation/master/log/stdout.log /tmp/dm_test/sync_collation/dmctl.1714131835.log /tmp/dm_test/sync_collation/worker2/log/dm-worker.log /tmp/dm_test/sync_collation/worker2/log/stdout.log /tmp/dm_test/sync_collation/dmctl.1714131830.log /tmp/dm_test/sync_collation/dmctl.1714131837.log /tmp/dm_test/sync_collation/dmctl.1714131829.log /tmp/dm_test/sync_collation/worker1/log/dm-worker.log /tmp/dm_test/sync_collation/worker1/log/stdout.log /tmp/dm_test/sync_collation/dmctl.1714131834.log /tmp/dm_test/sync_collation/dmctl.1714131836.log /tmp/dm_test/adjust_gtid/master/log/dm-master.log /tmp/dm_test/adjust_gtid/master/log/stdout.log /tmp/dm_test/adjust_gtid/worker2/log/dm-worker.log /tmp/dm_test/adjust_gtid/worker2/log/stdout.log /tmp/dm_test/adjust_gtid/dmctl.1714130193.log /tmp/dm_test/adjust_gtid/dmctl.1714130203.log /tmp/dm_test/adjust_gtid/dmctl.1714130192.log /tmp/dm_test/adjust_gtid/dmctl.1714130205.log /tmp/dm_test/adjust_gtid/sync_diff_stdout.log /tmp/dm_test/adjust_gtid/worker1/log/dm-worker.log /tmp/dm_test/adjust_gtid/worker1/log/stdout.log /tmp/dm_test/adjust_gtid/dmctl.1714130189.log /tmp/dm_test/adjust_gtid/dmctl.1714130191.log /tmp/dm_test/duplicate_event/dmctl.1714130643.log /tmp/dm_test/duplicate_event/master/log/dm-master.log /tmp/dm_test/duplicate_event/master/log/stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130642.log /tmp/dm_test/duplicate_event/worker2/log/dm-worker.log /tmp/dm_test/duplicate_event/worker2/log/stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130575.log /tmp/dm_test/duplicate_event/sync_diff_stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130641.log /tmp/dm_test/duplicate_event/worker1/log/dm-worker.log /tmp/dm_test/duplicate_event/worker1/log/stdout.log /tmp/dm_test/duplicate_event/dmctl.1714130640.log /tmp/dm_test/downstream_diff_index/master/log/dm-master.log /tmp/dm_test/downstream_diff_index/master/log/stdout.log /tmp/dm_test/downstream_diff_index/dmctl.1714130439.log /tmp/dm_test/downstream_diff_index/worker2/log/dm-worker.log /tmp/dm_test/downstream_diff_index/worker2/log/stdout.log /tmp/dm_test/downstream_diff_index/dmctl.1714130441.log /tmp/dm_test/downstream_diff_index/worker1/log/dm-worker.log /tmp/dm_test/downstream_diff_index/worker1/log/stdout.log /tmp/dm_test/downstream_diff_index/dmctl.1714130440.log /tmp/dm_test/binlog_parse/master/log/dm-master.log /tmp/dm_test/binlog_parse/master/log/stdout.log /tmp/dm_test/binlog_parse/dmctl.1714130258.log /tmp/dm_test/binlog_parse/dmctl.1714130256.log /tmp/dm_test/binlog_parse/sync_diff_stdout.log /tmp/dm_test/binlog_parse/dmctl.1714130259.log /tmp/dm_test/binlog_parse/dmctl.1714130257.log /tmp/dm_test/binlog_parse/worker1/log/dm-worker.log /tmp/dm_test/binlog_parse/worker1/log/stdout.log /tmp/dm_test/binlog_parse/dmctl.1714130260.log /tmp/dm_test/goroutines/stack/log/master-8261.log /tmp/dm_test/goroutines/stack/log/master-8361.log /tmp/dm_test/goroutines/stack/log/worker-8262.log /tmp/dm_test/goroutines/stack/log/master-8661.log /tmp/dm_test/goroutines/stack/log/worker-18263.log /tmp/dm_test/goroutines/stack/log/master-8461.log /tmp/dm_test/goroutines/stack/log/worker-18262.log /tmp/dm_test/goroutines/stack/log/master-8561.log /tmp/dm_test/goroutines/stack/log/worker-8264.log /tmp/dm_test/goroutines/stack/log/worker-8263.log /tmp/dm_test/goroutines/stack/log/master-8761.log /tmp/dm_test/fake_rotate_event/master/log/dm-master.log /tmp/dm_test/fake_rotate_event/master/log/stdout.log /tmp/dm_test/fake_rotate_event/dmctl.1714130705.log /tmp/dm_test/fake_rotate_event/sync_diff_stdout.log /tmp/dm_test/fake_rotate_event/worker1/log/dm-worker.log /tmp/dm_test/fake_rotate_event/worker1/log/stdout.log /tmp/dm_test/fake_rotate_event/dmctl.1714130710.log /tmp/dm_test/fake_rotate_event/dmctl.1714130706.log /tmp/dm_test/sql_mode/master/log/dm-master.log /tmp/dm_test/sql_mode/master/log/stdout.log /tmp/dm_test/sql_mode/dmctl.1714131815.log /tmp/dm_test/sql_mode/worker2/log/dm-worker.log /tmp/dm_test/sql_mode/worker2/log/stdout.log /tmp/dm_test/sql_mode/dmctl.1714131814.log /tmp/dm_test/sql_mode/sync_diff_stdout.log /tmp/dm_test/sql_mode/worker1/log/dm-worker.log /tmp/dm_test/sql_mode/worker1/log/stdout.log /tmp/dm_test/sql_mode/dmctl.1714131817.log /tmp/dm_test/ha_cases/master2/log/dm-master.log /tmp/dm_test/ha_cases/master2/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131069.log /tmp/dm_test/ha_cases/worker2/log/dm-worker.log /tmp/dm_test/ha_cases/worker2/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131064.log /tmp/dm_test/ha_cases/dmctl.1714131067.log /tmp/dm_test/ha_cases/sync_diff_stdout.log /tmp/dm_test/ha_cases/worker1/log/dm-worker.log /tmp/dm_test/ha_cases/worker1/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131072.log /tmp/dm_test/ha_cases/master1/log/dm-master.log /tmp/dm_test/ha_cases/master1/log/stdout.log /tmp/dm_test/ha_cases/dmctl.1714131070.log /tmp/dm_test/ha_cases/dmctl.1714131063.log /tmp/dm_test/ha_cases/dmctl.1714131068.log /tmp/dm_test/ha_cases/dmctl.1714131073.log /tmp/dm_test/ha_cases/master3/log/dm-master.log /tmp/dm_test/ha_cases/master3/log/stdout.log /tmp/dm_test/new_collation_off/master/log/dm-master.log /tmp/dm_test/new_collation_off/master/log/stdout.log /tmp/dm_test/new_collation_off/sync_diff_stdout.log /tmp/dm_test/new_collation_off/worker1/log/dm-worker.log /tmp/dm_test/new_collation_off/worker1/log/stdout.log /tmp/dm_test/new_collation_off/dmctl.1714131292.log /tmp/dm_test/sequence_sharding_optimistic/master/log/dm-master.log /tmp/dm_test/sequence_sharding_optimistic/master/log/stdout.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131666.log /tmp/dm_test/sequence_sharding_optimistic/remove_schema.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131656.log /tmp/dm_test/sequence_sharding_optimistic/set_schema.log /tmp/dm_test/sequence_sharding_optimistic/worker2/log/dm-worker.log /tmp/dm_test/sequence_sharding_optimistic/worker2/log/stdout.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131661.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131657.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131667.log /tmp/dm_test/sequence_sharding_optimistic/sync_diff_stdout.log /tmp/dm_test/sequence_sharding_optimistic/worker1/log/dm-worker.log /tmp/dm_test/sequence_sharding_optimistic/worker1/log/stdout.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131665.log /tmp/dm_test/sequence_sharding_optimistic/get_schema.log /tmp/dm_test/sequence_sharding_optimistic/dmctl.1714131664.log /tmp/dm_test/gbk/master/log/dm-master.log /tmp/dm_test/gbk/master/log/stdout.log /tmp/dm_test/gbk/dmctl.1714130833.log /tmp/dm_test/gbk/worker2/log/dm-worker.log /tmp/dm_test/gbk/worker2/log/stdout.log /tmp/dm_test/gbk/dmctl.1714130865.log /tmp/dm_test/gbk/dmctl.1714130836.log /tmp/dm_test/gbk/sync_diff_stdout.log /tmp/dm_test/gbk/worker1/log/dm-worker.log /tmp/dm_test/gbk/worker1/log/stdout.log /tmp/dm_test/gbk/dmctl.1714130796.log /tmp/dm_test/gbk/dmctl.1714130795.log /tmp/dm_test/dm_syncer/master/log/dm-master.log /tmp/dm_test/dm_syncer/master/log/stdout.log /tmp/dm_test/dm_syncer/worker2/log/dm-worker.log /tmp/dm_test/dm_syncer/worker2/log/stdout.log /tmp/dm_test/dm_syncer/dmctl.1714130422.log /tmp/dm_test/dm_syncer/syncer1/log/stdout.log /tmp/dm_test/dm_syncer/syncer1/log/dm-syncer.log /tmp/dm_test/dm_syncer/dmctl.1714130419.log /tmp/dm_test/dm_syncer/sync_diff_stdout.log /tmp/dm_test/dm_syncer/syncer2/log/stdout.log /tmp/dm_test/dm_syncer/syncer2/log/dm-syncer.log /tmp/dm_test/dm_syncer/worker1/log/dm-worker.log /tmp/dm_test/dm_syncer/worker1/log/stdout.log /tmp/dm_test/dm_syncer/dmctl.1714130420.log /tmp/dm_test/extend_column/master/log/dm-master.log /tmp/dm_test/extend_column/master/log/stdout.log /tmp/dm_test/extend_column/worker2/log/dm-worker.log /tmp/dm_test/extend_column/worker2/log/stdout.log /tmp/dm_test/extend_column/dmctl.1714130692.log /tmp/dm_test/extend_column/worker1/log/dm-worker.log /tmp/dm_test/extend_column/worker1/log/stdout.log /tmp/dm_test/extend_column/dmctl.1714130694.log /tmp/dm_test/openapi/master2/log/dm-master.log /tmp/dm_test/openapi/master2/log/stdout.log /tmp/dm_test/openapi/worker2/log/dm-worker.log /tmp/dm_test/openapi/worker2/log/stdout.log /tmp/dm_test/openapi/worker1/log/dm-worker.log /tmp/dm_test/openapi/worker1/log/stdout.log /tmp/dm_test/openapi/master1/log/dm-master.log /tmp/dm_test/openapi/master1/log/stdout.log /tmp/dm_test/shardddl_optimistic/master/log/dm-master.log /tmp/dm_test/shardddl_optimistic/master/log/stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131760.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131707.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131713.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131706.log /tmp/dm_test/shardddl_optimistic/worker2/log/dm-worker.log /tmp/dm_test/shardddl_optimistic/worker2/log/stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131711.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131717.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131761.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131705.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131772.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131777.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131781.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131709.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131712.log /tmp/dm_test/shardddl_optimistic/sync_diff_stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131779.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131764.log /tmp/dm_test/shardddl_optimistic/worker1/log/dm-worker.log /tmp/dm_test/shardddl_optimistic/worker1/log/stdout.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131762.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131766.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131710.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131773.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131768.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131785.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131716.log /tmp/dm_test/shardddl_optimistic/dmctl.1714131769.log /tmp/dm_test/foreign_key/master/log/dm-master.log /tmp/dm_test/foreign_key/master/log/stdout.log /tmp/dm_test/foreign_key/dmctl.1714130720.log /tmp/dm_test/foreign_key/sync_diff_stdout.log /tmp/dm_test/foreign_key/worker1/log/dm-worker.log /tmp/dm_test/foreign_key/worker1/log/stdout.log /tmp/dm_test/foreign_key/dmctl.1714130718.log /tmp/dm_test/drop_column_with_index/master/log/dm-master.log /tmp/dm_test/drop_column_with_index/master/log/stdout.log /tmp/dm_test/drop_column_with_index/dmctl.1714130469.log /tmp/dm_test/drop_column_with_index/dmctl.1714130467.log /tmp/dm_test/drop_column_with_index/dmctl.1714130468.log /tmp/dm_test/drop_column_with_index/dmctl.1714130470.log /tmp/dm_test/drop_column_with_index/sync_diff_stdout.log /tmp/dm_test/drop_column_with_index/worker1/log/dm-worker.log /tmp/dm_test/drop_column_with_index/worker1/log/stdout.log /tmp/dm_test/drop_column_with_index/dmctl.1714130475.log /tmp/dm_test/expression_filter/master/log/dm-master.log /tmp/dm_test/expression_filter/master/log/stdout.log /tmp/dm_test/expression_filter/dmctl.1714130664.log /tmp/dm_test/expression_filter/worker1/log/dm-worker.log /tmp/dm_test/expression_filter/worker1/log/stdout.log /tmp/dm_test/expression_filter/dmctl.1714130670.log /tmp/dm_test/expression_filter/dmctl.1714130663.log /tmp/dm_test/slow_relay_writer/dmctl.1714131794.log /tmp/dm_test/slow_relay_writer/master/log/dm-master.log /tmp/dm_test/slow_relay_writer/master/log/stdout.log /tmp/dm_test/slow_relay_writer/dmctl.1714131806.log /tmp/dm_test/slow_relay_writer/sync_diff_stdout.log /tmp/dm_test/slow_relay_writer/worker1/log/dm-worker.log /tmp/dm_test/slow_relay_writer/worker1/log/stdout.log /tmp/dm_test/slow_relay_writer/dmctl.1714131793.log /tmp/dm_test/tracker_ignored_ddl/master/log/dm-master.log /tmp/dm_test/tracker_ignored_ddl/master/log/stdout.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131846.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131852.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131849.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131850.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131855.log /tmp/dm_test/tracker_ignored_ddl/sync_diff_stdout.log /tmp/dm_test/tracker_ignored_ddl/worker1/log/dm-worker.log /tmp/dm_test/tracker_ignored_ddl/worker1/log/stdout.log /tmp/dm_test/tracker_ignored_ddl/dmctl.1714131856.log /tmp/dm_test/metrics/master/log/dm-master.log /tmp/dm_test/metrics/master/log/stdout.log /tmp/dm_test/metrics/dmctl.1714131255.log /tmp/dm_test/metrics/worker2/log/dm-worker.log /tmp/dm_test/metrics/worker2/log/stdout.log /tmp/dm_test/metrics/dmctl.1714131256.log /tmp/dm_test/metrics/dmctl.1714131254.log /tmp/dm_test/metrics/dmctl.1714131280.log /tmp/dm_test/metrics/dmctl.1714131252.log /tmp/dm_test/metrics/dmctl.1714131277.log /tmp/dm_test/metrics/sync_diff_stdout.log /tmp/dm_test/metrics/dmctl.1714131260.log /tmp/dm_test/metrics/worker1/log/dm-worker.log /tmp/dm_test/metrics/worker1/log/stdout.log /tmp/dm_test/checkpoint_transaction/master/log/dm-master.log /tmp/dm_test/checkpoint_transaction/master/log/stdout.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130364.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130333.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130334.log /tmp/dm_test/checkpoint_transaction/sync_diff_stdout.log /tmp/dm_test/checkpoint_transaction/worker1/log/dm-worker.log /tmp/dm_test/checkpoint_transaction/worker1/log/stdout.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130379.log /tmp/dm_test/checkpoint_transaction/dmctl.1714130382.log /tmp/dm_test/incompatible_ddl_changes/master/log/dm-master.log /tmp/dm_test/incompatible_ddl_changes/master/log/stdout.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132563.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132556.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132558.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132562.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132561.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132560.log /tmp/dm_test/incompatible_ddl_changes/worker1/log/dm-worker.log /tmp/dm_test/incompatible_ddl_changes/worker1/log/stdout.log /tmp/dm_test/incompatible_ddl_changes/dmctl.1714132557.log /tmp/dm_test/downstream/tidb/log/tidb.log /tmp/dm_test/http_proxies/master/log/dm-master.log /tmp/dm_test/http_proxies/master/log/stdout.log /tmp/dm_test/http_proxies/dmctl.1714131095.log /tmp/dm_test/http_proxies/dmctl.1714131084.log /tmp/dm_test/http_proxies/worker1/log/dm-worker.log /tmp/dm_test/http_proxies/worker1/log/stdout.log /tmp/dm_test/http_proxies/dmctl.1714131106.log /tmp/dm_test/only_dml/dmctl.1714131341.log /tmp/dm_test/only_dml/dmctl.1714131332.log /tmp/dm_test/only_dml/dmctl.1714131337.log /tmp/dm_test/only_dml/master/log/dm-master.log /tmp/dm_test/only_dml/master/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714131302.log /tmp/dm_test/only_dml/dmctl.1714131325.log /tmp/dm_test/only_dml/dmctl.1714131342.log /tmp/dm_test/only_dml/dmctl.1714131328.log /tmp/dm_test/only_dml/dmctl.1714131308.log /tmp/dm_test/only_dml/dmctl.1714131338.log /tmp/dm_test/only_dml/dmctl.1714131326.log /tmp/dm_test/only_dml/dmctl.1714131329.log /tmp/dm_test/only_dml/dmctl.1714131305.log /tmp/dm_test/only_dml/dmctl.1714131314.log /tmp/dm_test/only_dml/worker2/log/dm-worker.log /tmp/dm_test/only_dml/worker2/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714131339.log /tmp/dm_test/only_dml/dmctl.1714131333.log /tmp/dm_test/only_dml/dmctl.1714131301.log /tmp/dm_test/only_dml/dmctl.1714131315.log /tmp/dm_test/only_dml/dmctl.1714131307.log /tmp/dm_test/only_dml/dmctl.1714131321.log /tmp/dm_test/only_dml/dmctl.1714131311.log /tmp/dm_test/only_dml/dmctl.1714131345.log /tmp/dm_test/only_dml/dmctl.1714131336.log /tmp/dm_test/only_dml/dmctl.1714131343.log /tmp/dm_test/only_dml/dmctl.1714131340.log /tmp/dm_test/only_dml/dmctl.1714131334.log /tmp/dm_test/only_dml/sync_diff_stdout.log /tmp/dm_test/only_dml/dmctl.1714131331.log /tmp/dm_test/only_dml/dmctl.1714131324.log /tmp/dm_test/only_dml/dmctl.1714131306.log /tmp/dm_test/only_dml/dmctl.1714131309.log /tmp/dm_test/only_dml/worker1/log/dm-worker.log /tmp/dm_test/only_dml/worker1/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714131312.log /tmp/dm_test/only_dml/dmctl.1714131330.log /tmp/dm_test/only_dml/dmctl.1714131313.log /tmp/dm_test/only_dml/dmctl.1714131304.log /tmp/dm_test/only_dml/dmctl.1714131323.log /tmp/dm_test/only_dml/dmctl.1714131322.log /tmp/dm_test/only_dml/dmctl.1714131320.log /tmp/dm_test/only_dml/dmctl.1714131319.log /tmp/dm_test/only_dml/dmctl.1714131317.log /tmp/dm_test/only_dml/dmctl.1714131316.log /tmp/dm_test/full_mode/master/log/dm-master.log /tmp/dm_test/full_mode/master/log/stdout.log /tmp/dm_test/full_mode/dmctl.1714130782.log /tmp/dm_test/full_mode/dmctl.1714130783.log /tmp/dm_test/full_mode/worker2/log/dm-worker.log /tmp/dm_test/full_mode/worker2/log/stdout.log /tmp/dm_test/full_mode/dmctl.1714130785.log /tmp/dm_test/full_mode/sync_diff_stdout.log /tmp/dm_test/full_mode/worker1/log/dm-worker.log /tmp/dm_test/full_mode/worker1/log/stdout.log /tmp/dm_test/full_mode/dmctl.1714130784.log /tmp/dm_test/async_checkpoint_flush/master/log/dm-master.log /tmp/dm_test/async_checkpoint_flush/master/log/stdout.log /tmp/dm_test/async_checkpoint_flush/dmctl.1714130217.log /tmp/dm_test/async_checkpoint_flush/sync_diff_stdout.log /tmp/dm_test/async_checkpoint_flush/worker1/log/dm-worker.log /tmp/dm_test/async_checkpoint_flush/worker1/log/stdout.log + ls -alh log-G11.tar.gz -rw-r--r--. 1 jenkins jenkins 5.6M Apr 26 19:56 log-G11.tar.gz [Pipeline] archiveArtifacts Archiving artifacts [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 2 Finished: FAILURE