Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_dm_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is due for a refresh after 60 minutes, clearing. Caching library tipipeline@main Attempting to resolve main from remote references... > git --version # timeout=10 > git --version # 'git version 2.39.2' > git ls-remote -- https://github.com/PingCAP-QE/ci # timeout=10 Found match: refs/heads/main revision 03312178c534dce949face80c69812d989e55009 The recommended git tool is: git No credentials specified > git rev-parse --resolve-git-dir /var/jenkins_home/jobs/pingcap/jobs/tiflow/jobs/pull_dm_integration_test/workspace@libs/028688d6a2298cb8f56fc89656584a6e89655f25aed21ceab3b5624e60735994/.git # timeout=10 Fetching changes from the remote Git repository > git config remote.origin.url https://github.com/PingCAP-QE/ci # timeout=10 Fetching without tags Fetching upstream changes from https://github.com/PingCAP-QE/ci > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --no-tags --force --progress -- https://github.com/PingCAP-QE/ci +refs/heads/*:refs/remotes/origin/* # timeout=10 Checking out Revision 03312178c534dce949face80c69812d989e55009 (main) > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 Agent pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 is provisioned from template pingcap_tiflow_pull_dm_integration_test_1907-s2xpw-8dmtw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5423625e5e880c2fec940a05ce2ada4518e0e469" jenkins/label: "pingcap_tiflow_pull_dm_integration_test_1907-s2xpw" name: "pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 + printenv PROW_JOB_ID=4dea3dab-983c-449a-a192-79545a642f4d JENKINS_NODE_COOKIE=9e60bf74-891e-488a-b61c-c9705c8b8548 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_dm_integration_test-1907 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_dm_integration_test","buildid":"1787100770022199296","prowjobid":"4dea3dab-983c-449a-a192-79545a642f4d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"76ad8e6dbaeacfb7ca050ed5db5eba0401b203cf","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/76ad8e6dbaeacfb7ca050ed5db5eba0401b203cf","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_dm_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1907 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787100770022199296 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_dm_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_dm_integration_test_1907-s2xpw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-e4960c3787c8b0a028d56044159150f6020eb6e83c507357bbc62d29b344724b NODE_LABELS=pingcap_tiflow_pull_dm_integration_test_1907-s2xpw pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/display/redirect BUILD_NUMBER=1907 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build4254407797=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 -c golang -- bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-dm-integration-test-1907-s2xpw-8dmtw-qkrr7 -c golang -- bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 47911 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 5ddecfbb38ab8f81 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sun May 05 12:43:46 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, pkg/sink/codec/open/open_protocol_decoder.go, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/kafka_simple_handle_key_only_avro/conf/diff_config.toml, tests/integration_tests/run_group.sh] [Pipeline] echo diff file matched: pkg/applier/redo.go [Pipeline] echo matched, some diff files full path start with dm/ or pkg/ or go.mod, run the dm integration test [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-76ad8e6) 203833344 bytes in 2.61 secs (78129291 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/.git/ .git HEAD is now at 76ad8e6db fix tests POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/10919/head -> origin/pr/10919/head Previous HEAD position was 76ad8e6db fix tests HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..76ad8e6db Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 47 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 89 +++-- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 299 +++++++++++++++- pkg/applier/redo_test.go | 379 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ pkg/sink/codec/open/open_protocol_decoder.go | 2 + .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 +++ .../integration_tests/force_replicate_table/run.sh | 4 +- .../conf/diff_config.toml | 2 +- tests/integration_tests/run_group.sh | 2 +- 25 files changed, 1004 insertions(+), 106 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh ๐Ÿงพ Pre-merged result: 76ad8e6dbaeacfb7ca050ed5db5eba0401b203cf 76ad8e6db fix tests 9c6b3687c only split update event when handle changed 0de8dc3e4 fix test again โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-be15534-76ad8e6 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./dm/tests/download-integration-test-binaries.sh master % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 891 0 --:--:-- --:--:-- --:--:-- 911 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 907 0 --:--:-- --:--:-- --:--:-- 911 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 735 0 --:--:-- --:--:-- --:--:-- 745 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1063 0 --:--:-- --:--:-- --:--:-- 1078 Download binaries... >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-05 20:44:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-05 20:44:42 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-05 20:45:28 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tidb-tools.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/d28ee6cd546e41862cf361e7a35552fb517b883e/centos7/tidb-tools.tar.gz 2024-05-05 20:45:48 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/d28ee6cd546e41862cf361e7a35552fb517b883e/centos7/tidb-tools.tar.gz [185439936/185439936] -> "tmp/tidb-tools.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-05 20:45:53 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download gh-ost-binary-linux-20200828140552.tar.gz from https://github.com/github/gh-ost/releases/download/v1.1.0/gh-ost-binary-linux-20200828140552.tar.gz 2024-05-05 20:45:55 URL:https://objects.githubusercontent.com/github-production-release-asset-2e65be/54378638/ac8d1400-e94a-11ea-9664-8b516d434420?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240505%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240505T124554Z&X-Amz-Expires=300&X-Amz-Signature=a22f6362d5e677e6da59baf327b2224f42d757ef2e851bdf4b8660e9f4e97616&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=54378638&response-content-disposition=attachment%3B%20filename%3Dgh-ost-binary-linux-20200828140552.tar.gz&response-content-type=application%2Foctet-stream [5141587/5141587] -> "tmp/gh-ost-binary-linux-20200828140552.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.2G drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:45 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:45 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 20:45 bin -rwxr-xr-x. 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 127M Apr 8 13:50 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/gh-ost ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.2G drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:45 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:45 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 20:45 bin -rwxr-xr-x. 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 127M Apr 8 13:50 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (binary/pingcap/tiflow/dm-integration-test/rev-be15534-76ad8e6) 1506043904 bytes in 3.63 secs (414944341 bytes/sec) [Pipeline] { [Pipeline] sh + [[ ! -f bin/dm-master.test ]] + [[ ! -f bin/dm-test-tools/check_master_online ]] + [[ ! -f bin/dm-test-tools/check_worker_online ]] + echo 'Binaries already exist, skipping build...' Binaries already exist, skipping build... + ls -alh ./bin total 1.1G drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:45 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:45 .. -rwxr-xr-x. 1 jenkins jenkins 299M May 5 17:22 dm-master.test -rwxr-xr-x. 1 jenkins jenkins 285M May 5 17:25 dm-syncer.test drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 20:45 dm-test-tools -rwxr-xr-x. 1 jenkins jenkins 296M May 5 17:22 dm-worker.test -rwxr-xr-x. 1 jenkins jenkins 215M May 5 17:25 dmctl.test + ls -alh ./bin/dm-test-tools total 344M drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 20:45 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:45 .. -rwxr-xr-x. 1 jenkins jenkins 205M May 5 17:26 check_exit_safe_binlog -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_master_http_apis -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_master_online -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_master_online_http -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_worker_online + which ./bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test + which ./bin/dm-syncer.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test + which ./bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test + which ./bin/dmctl.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dmctl.test + which ./bin/dm-test-tools/check_master_online /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_master_online + which ./bin/dm-test-tools/check_worker_online /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_worker_online [Pipeline] } Cache not saved (binary/pingcap/tiflow/dm-integration-test/rev-be15534-76ad8e6 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/gh-ost ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 2.2G drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:46 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:45 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 20:46 bin -rwxr-xr-x. 1 jenkins jenkins 299M May 5 17:22 dm-master.test -rwxr-xr-x. 1 jenkins jenkins 285M May 5 17:25 dm-syncer.test drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 20:45 dm-test-tools -rwxr-xr-x. 1 jenkins jenkins 296M May 5 17:22 dm-worker.test -rwxr-xr-x. 1 jenkins jenkins 215M May 5 17:25 dmctl.test -rwxr-xr-x. 1 jenkins jenkins 9.8M May 5 20:46 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M May 5 20:46 minio -rwxr-xr-x. 1 jenkins jenkins 37M May 5 20:46 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 5 20:46 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 5 20:46 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 5 20:46 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 5 20:46 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 5 20:46 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 5 20:46 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 5 20:46 regions-dump -rwxr-xr-x. 1 jenkins jenkins 32M May 5 20:46 stores-dump -rwxr-xr-x. 1 jenkins jenkins 127M May 5 20:46 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 5 20:46 tidb-server -rwxr-xr-x. 1 jenkins jenkins 418M May 5 20:46 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 5 20:46 xprog + ls -alh ./bin/dm-test-tools total 344M drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 20:45 . drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:46 .. -rwxr-xr-x. 1 jenkins jenkins 205M May 5 17:26 check_exit_safe_binlog -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_master_http_apis -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_master_online -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_master_online_http -rwxr-xr-x. 1 jenkins jenkins 35M May 5 17:26 check_worker_online [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 53.32 secs (54804620 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt Created Pod: kubernetes jenkins-tiflow/dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125-z7wd4-ngg6k Created Pod: kubernetes jenkins-tiflow/dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t-2f8qw Created Pod: kubernetes jenkins-tiflow/dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459-qdxb7-j78zh Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14 Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. Created Pod: kubernetes jenkins-tiflow/dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1-s2stm-mr9gq [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e-bsb0q-s613t Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n Agent dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t-2f8qw is provisioned from template dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2b25238ff7c6dd83271f68b02111d5a23299df3b" jenkins/label: "dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5" name: "dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t-2f8qw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t-2f8qw" - name: "JENKINS_NAME" value: "dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t-2f8qw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t-2f8qw in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Agent dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125-z7wd4-ngg6k is provisioned from template dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125-z7wd4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d629c6e6a7547e4e503d1a7ee47d134e7f332bb2" jenkins/label: "dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125" name: "dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125-z7wd4-ngg6k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125-z7wd4-ngg6k" - name: "JENKINS_NAME" value: "dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125-z7wd4-ngg6k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-2bec69c1-09ba-4f6b-aa92-dfd1ae230125-z7wd4-ngg6k in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { Agent dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14 is provisioned from template dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ed01379b64755e69bd72570aa8fe98b28315c1b7" jenkins/label: "dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7" name: "dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14" - name: "JENKINS_NAME" value: "dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@8f5ceb0; decorates RemoteLauncher[hudson.remoting.Channel@6bae6921:JNLP4-connect connection from 10.233.72.209/10.233.72.209:42700] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1-s2stm-mr9gq is provisioned from template dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1-s2stm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7152e8394f40e66205a5fac4fef45679a314031c" jenkins/label: "dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1" name: "dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1-s2stm-mr9gq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1-s2stm-mr9gq" - name: "JENKINS_NAME" value: "dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1-s2stm-mr9gq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-1cb6e5d3-e78a-4494-83b6-cad9fcc32ee1-s2stm-mr9gq in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@48b51c42; decorates RemoteLauncher[hudson.remoting.Channel@52c8282:JNLP4-connect connection from 10.233.69.246/10.233.69.246:33942] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] podTemplate Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] node [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3b175989; decorates RemoteLauncher[hudson.remoting.Channel@16ea6b68:JNLP4-connect connection from 10.233.67.219/10.233.67.219:34938] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@18b2bbb2; decorates RemoteLauncher[hudson.remoting.Channel@2eab5d1a:JNLP4-connect connection from 10.233.70.204/10.233.70.204:33264] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e-z30ss-8hmvf Created Pod: kubernetes jenkins-tiflow/dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0-z2z80-1tsx9 Agent dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459-qdxb7-j78zh is provisioned from template dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459-qdxb7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1ba7dfe1812aeb997a3c2c66848df12ae629abc4" jenkins/label: "dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459" name: "dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459-qdxb7-j78zh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459-qdxb7-j78zh" - name: "JENKINS_NAME" value: "dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459-qdxb7-j78zh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-5f403869-cd3d-4e6b-8b01-2f94847a1459-qdxb7-j78zh in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Created Pod: kubernetes jenkins-tiflow/dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4-xn0t6-r7tht Created Pod: kubernetes jenkins-tiflow/dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg-pwq5f Created Pod: kubernetes jenkins-tiflow/dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@deed1; decorates RemoteLauncher[hudson.remoting.Channel@6d411e74:JNLP4-connect connection from 10.233.66.219/10.233.66.219:48396] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n is provisioned from template dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "efd07ca501277bec52b6ba427e3450e423c7eeac" jenkins/label: "dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751" name: "dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n" - name: "JENKINS_NAME" value: "dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Avoid second fetch Running on dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e-bsb0q-s613t is provisioned from template dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e-bsb0q --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9ec51468f52a8dd1b42651c5952de3cac2c0ae1e" jenkins/label: "dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e" name: "dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e-bsb0q-s613t" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e-bsb0q-s613t" - name: "JENKINS_NAME" value: "dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e-bsb0q-s613t" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on dm-it-f11b9ad6-01de-4623-97ea-2bec733d4a5e-bsb0q-s613t in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] checkout [Pipeline] withEnv [Pipeline] { The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 5 12:47 . drwxr-sr-x. 17 1000 1000 4.0K May 5 12:47 .. -rw-------. 1 root 1000 1.7K May 5 12:47 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 ca.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 client-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 client-key.pem -rw-------. 1 root 1000 1.7K May 5 12:47 private_key.pem -rw-r--r--. 1 root 1000 451 May 5 12:47 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 server-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 server-key.pem [Pipeline] container [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] { [Pipeline] cache Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@40323896; decorates RemoteLauncher[hudson.remoting.Channel@2767b461:JNLP4-connect connection from 10.233.97.254/10.233.97.254:51706] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@628df133; decorates RemoteLauncher[hudson.remoting.Channel@7f300bc6:JNLP4-connect connection from 10.233.107.134/10.233.107.134:55398] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Still waiting to schedule task Waiting for next available executor on โ€˜dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqtโ€™ > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e-z30ss-8hmvf is provisioned from template dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e-z30ss --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e87b72383ff6fba0a6013b022acf338a642f4120" jenkins/label: "dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e" name: "dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e-z30ss-8hmvf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e-z30ss-8hmvf" - name: "JENKINS_NAME" value: "dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e-z30ss-8hmvf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-c1d07d4d-d948-4c99-9632-bb4fc13d1f5e-z30ss-8hmvf in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Still waiting to schedule task โ€˜dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg-pwq5fโ€™ is offline Agent dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4-xn0t6-r7tht is provisioned from template dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4-xn0t6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5c687109595fe255dfae9ba3eab74c5a94ef8e0c" jenkins/label: "dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4" name: "dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4-xn0t6-r7tht" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4-xn0t6-r7tht" - name: "JENKINS_NAME" value: "dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4-xn0t6-r7tht" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-f44ba49c-e323-4175-b1a9-c9bb0d4b29b4-xn0t6-r7tht in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Agent dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg-pwq5f is provisioned from template dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9fa3f2ad42999b52916e0bcd242bccc7573fd724" jenkins/label: "dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d" name: "dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg-pwq5f" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg-pwq5f" - name: "JENKINS_NAME" value: "dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg-pwq5f" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-d4a5440c-c2e8-4896-82d4-7914c1dce19d-1x8qg-pwq5f in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Agent dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25 is provisioned from template dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc Agent dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0-z2z80-1tsx9 is provisioned from template dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0-z2z80 --- apiVersion: "v1" kind: "Pod" metadata: annotations: --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" jenkins/label-digest: "da2073b0c6120ca290c1690f77e75bbd91100dbd" jenkins/label: "dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0" labels: jenkins/jenkins-jenkins-agent: "true" name: "dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0-z2z80-1tsx9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: jenkins/label-digest: "94f977256db8e7f68d4d9a1b5f158bcda1169c1f" jenkins/label: "dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" name: "dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: operator: "In" values: - "amd64" containers: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" - key: "kubernetes.io/arch" operator: "In" values: - "amd64" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" name: "golang" resources: limits: memory: "16Gi" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" cpu: "6" requests: memory: "12Gi" cpu: "4" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" name: "mysql1" resources: limits: memory: "4Gi" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" memory: "4Gi" cpu: "2" tty: true volumeMounts: cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" memory: "128Mi" cpu: "100m" tty: true volumeMounts: value: "dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0-z2z80-1tsx9" - name: "JENKINS_NAME" value: "dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0-z2z80-1tsx9" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: memory: "256Mi" cpu: "100m" tty: true volumeMounts: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" name: "workspace-volume" value: "dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25" - name: "JENKINS_NAME" value: "dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Running on dm-it-a633d7b6-6bb7-4df6-b1cc-0a98d81ac4e0-z2z80-1tsx9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Agent dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt is provisioned from template dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1907/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "677f715dd3920018f4c9957cf4f294edff54b25b" jenkins/label: "dm-it-64f08141-f09f-467f-bf00-09430931b325" name: "dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt" - name: "JENKINS_NAME" value: "dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 13.28 secs (220007684 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] sh [Pipeline] stage [Pipeline] { (Test) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:47 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:47 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 5 20:47 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:47 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 .github -rw-r--r--. 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:47 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:47 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 20:47 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:47 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 5 20:47 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:47 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:47 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 examples -rw-r--r--. 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 5 20:47 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:47 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:47 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:47 tests -rw-r--r--. 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:47 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4aae8833; decorates RemoteLauncher[hudson.remoting.Channel@63940387:JNLP4-connect connection from 10.233.100.160/10.233.100.160:36004] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3a4b427; decorates RemoteLauncher[hudson.remoting.Channel@487907fc:JNLP4-connect connection from 10.233.108.86/10.233.108.86:46578] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1f8308c2; decorates RemoteLauncher[hudson.remoting.Channel@835dc02:JNLP4-connect connection from 10.233.90.138/10.233.90.138:33832] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3058e271; decorates RemoteLauncher[hudson.remoting.Channel@2de50544:JNLP4-connect connection from 10.233.106.213/10.233.106.213:60112] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@ee15f0c; decorates RemoteLauncher[hudson.remoting.Channel@4d51fc5f:JNLP4-connect connection from 10.233.127.107/10.233.127.107:34646] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7bf156a8; decorates RemoteLauncher[hudson.remoting.Channel@3ff0acbd:JNLP4-connect connection from 10.233.71.26/10.233.71.26:44376] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 5 12:47 . drwxr-sr-x. 17 1000 1000 4.0K May 5 12:47 .. -rw-------. 1 root 1000 1.7K May 5 12:47 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 ca.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 client-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 client-key.pem -rw-------. 1 root 1000 1.7K May 5 12:47 private_key.pem -rw-r--r--. 1 root 1000 451 May 5 12:47 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 server-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 server-key.pem Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // timeout Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 5 12:47 . drwxr-sr-x. 17 1000 1000 4.0K May 5 12:47 .. -rw-------. 1 root 1000 1.7K May 5 12:47 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 ca.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 client-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 client-key.pem -rw-------. 1 root 1000 1.7K May 5 12:47 private_key.pem -rw-r--r--. 1 root 1000 451 May 5 12:47 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 server-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 server-key.pem [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh + '[' TLS_GROUP == G03 ']' + echo 'run G03 test' run G03 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G03 [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 5 12:47 . drwxr-sr-x. 17 1000 1000 4.0K May 5 12:47 .. -rw-------. 1 root 1000 1.7K May 5 12:47 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 ca.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 client-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 client-key.pem -rw-------. 1 root 1000 1.7K May 5 12:47 private_key.pem -rw-r--r--. 1 root 1000 451 May 5 12:47 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 server-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 server-key.pem + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 5 12:47 . drwxr-sr-x. 17 1000 1000 4.0K May 5 12:47 .. -rw-------. 1 root 1000 1.7K May 5 12:47 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 ca.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 client-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 client-key.pem -rw-------. 1 root 1000 1.7K May 5 12:47 private_key.pem -rw-r--r--. 1 root 1000 451 May 5 12:47 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:47 server-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:47 server-key.pem [Pipeline] sh [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 5 12:47 . drwxr-sr-x 17 1000 1000 4.0K May 5 12:47 .. -rw------- 1 root 1000 1.7K May 5 12:47 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:47 ca.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:47 client-cert.pem -rw------- 1 root 1000 1.7K May 5 12:47 client-key.pem -rw------- 1 root 1000 1.7K May 5 12:47 private_key.pem -rw-r--r-- 1 root 1000 451 May 5 12:47 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:47 server-cert.pem -rw------- 1 root 1000 1.7K May 5 12:47 server-key.pem [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G03 Run cases: dmctl_advance dmctl_basic dmctl_command downstream_diff_index downstream_more_column ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 5 12:47 . drwxr-sr-x 17 1000 1000 4.0K May 5 12:47 .. -rw------- 1 root 1000 1.7K May 5 12:47 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:47 ca.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:47 client-cert.pem -rw------- 1 root 1000 1.7K May 5 12:47 client-key.pem -rw------- 1 root 1000 1.7K May 5 12:47 private_key.pem -rw-r--r-- 1 root 1000 451 May 5 12:47 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:47 server-cert.pem -rw------- 1 root 1000 1.7K May 5 12:47 server-key.pem [Pipeline] } [Pipeline] } [Pipeline] } Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] // container Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] cache Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [dmctl_advance] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm [Sun May 5 20:47:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock unlock" dmctl test cmd: "shard-ddl-lock unlock lock_id --force-remove=invalid-force-remove" dmctl test cmd: "handle-error" dmctl test cmd: "handle-error test-task --binlog-pos mysql-bin:shoud-bin-digital skip" dmctl test cmd: "handle-error test-task replace alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "handle-error test-task wrong_operation" dmctl test cmd: "binlog" dmctl test cmd: "binlog skip test-task --binlog-pos mysql-bin:should-be-digital" dmctl test cmd: "binlog replace test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog inject test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog list test-task --binlog-pos mysql-bin:should-be-digital" dmctl test cmd: "binlog wrong_operation test-task" dmctl test cmd: "binlog-schema" dmctl test cmd: "binlog-schema test" dmctl test cmd: "binlog-schema list test" dmctl test cmd: "binlog-schema update test" dmctl test cmd: "binlog-schema update test test_db test_table --from-source" dmctl test cmd: "binlog-schema update test test_db test_table --from-source --from-target" dmctl test cmd: "binlog-schema update test test_db test_table schema.sql --from-source" dmctl test cmd: "binlog-schema delete test" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:47:41 CST 2024] <<<<<< test case dmctl_advance success! >>>>>> start running case: [dmctl_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm dmctl test cmd: "any command" dmctl test cmd: "any command" [Sun May 5 20:47:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive check_task_wrong_arg dmctl test cmd: "check-task" dmctl test cmd: "check-task not_exists_config_file" pause_relay_wrong_arg dmctl test cmd: "pause-relay wrong_arg" dmctl test cmd: "pause-relay" resume_relay_wrong_arg dmctl test cmd: "resume-relay wrong_arg" dmctl test cmd: "resume-relay" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 9.43 secs (309919337 bytes/sec) [Pipeline] { [Pipeline] cache pause_task_wrong_arg dmctl test cmd: "pause-task" dmctl test cmd: "pause-task -s a -s b" resume_task_wrong_arg dmctl test cmd: "resume-task" dmctl test cmd: "resume-task -s a -s b" query_status_wrong_arg dmctl test cmd: "query-status wrong_args_count more_than_one" dmctl test cmd: "query-status -s source-x task-y" start_task_wrong_arg dmctl test cmd: "start-task" dmctl test cmd: "start-task not_exists_config_file" stop_task_wrong_arg dmctl test cmd: "stop-task" dmctl test cmd: "stop-task -s a -s b" show_ddl_locks_wrong_arg dmctl test cmd: "shard-ddl-lock a b" dmctl test cmd: "show-ddl-locks a b" purge_relay_wrong_arg dmctl test cmd: "purge-relay wrong_arg" dmctl test cmd: "purge-relay" dmctl test cmd: "purge-relay --filename bin-000001 -s mysql-replica-01 -s mysql-replica-02" operate_source_empty_arg dmctl test cmd: "operate-source" dmctl test cmd: "operate-source create not_exists_config_file" dmctl test cmd: "operate-source invalid /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" dmctl test cmd: "operate-source stop /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" transfer_source_empty_arg dmctl test cmd: "transfer-source" dmctl test cmd: "transfer-source source-id" dmctl test cmd: "transfer-source source-id worker-id another-worker" start_relay_empty_arg dmctl test cmd: "start-relay" dmctl test cmd: "start-relay wrong_arg" stop_relay_empty_arg dmctl test cmd: "stop-relay" dmctl test cmd: "stop-relay wrong_arg" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" source-id: mysql-replica-01 server-id: 111111111 enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' enable-relay: false from: host: 127.0.0.1 user: root port: 3306 filters: - schema-pattern: "dmctl" table-pattern: "t_1" sql-pattern: ["alter table .* add column aaa int"] action: Ignore - schema-pattern: "dmctl" table-pattern: "t_2" sql-pattern: ["alter table .* add column aaa int"] action: Ignore dmctl test cmd: "operate-source stop /tmp/dm_test/dmctl_basic/source1-wrong.yaml mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" [Sun May 5 20:47:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" dmctl test cmd: "operate-source show" [Sun May 5 20:47:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" dmctl test cmd: "operate-source show -s mysql-replica-01" ############################################################ test case for operate tasks bound to a source ############################################################ dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/foo-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" dmctl test cmd: "pause-task -s xxx" dmctl test cmd: "resume-task -s xxx" dmctl test cmd: "stop-task -s xxx" dmctl test cmd: "pause-task -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 11.05 secs (264430603 bytes/sec) [Pipeline] { [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] cache dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/baz-task.yaml" dmctl test cmd: "pause-task -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 6.79 secs (430106100 bytes/sec) [Pipeline] { [Pipeline] cache dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "transfer-source mysql-replica-01 worker1" dmctl test cmd: "transfer-source mysql-replica-01 worker2" kill worker2 wait process worker2 exit... wait process worker2 exit... process worker2 already exit dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" restart worker2 [Sun May 5 20:48:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-01 worker2" dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "stop-relay -s mysql-replica-01" pause_relay_success dmctl test cmd: "pause-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=2 expected=2 dmctl test cmd: "resume-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl_check_task dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 100 -w 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 0" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" check_task_optimistic dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task4.yaml" check_task_only_warning dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" check task with empty unit config dmctl test cmd: "check-task /tmp/empty-dump.yaml" --- name: empty-unit-task task-mode: all target-database: host: "127.0.0.1" port: 4000 user: "root" password: "" mysql-instances: - source-id: "mysql-replica-01" mydumper-config-name: "global" loader-config-name: "global" syncer-config-name: "global" block-allow-list: "instance" mydumpers: global: threads: 4 loaders: global: syncers: global: worker-count: 32 block-allow-list: instance: do-dbs: ["dmctl"] do-tables: - db-name: "dmctl" tbl-name: "~^t_[\\d]+"dmctl test cmd: "check-task /tmp/empty-load.yaml" dmctl test cmd: "check-task /tmp/empty-sync.yaml" start task with empty unit config dmctl test cmd: "start-task /tmp/empty-cfg.yaml" dmctl test cmd: "config task empty-unit-task" dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/empty-unit-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/dmctl_basic/dm-task-error-database-config.yaml" dmctl_start_task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" dmctl test cmd: "query-status test_incremental_no_source_meta" got=3 expected=3 dmctl test cmd: "stop-task test_incremental_no_source_meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml " check diff successfully dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" wait process dmctl_basic/worker1 exit... wait process dmctl_basic/worker1 exit... process dmctl_basic/worker1 already exit wait process dmctl_basic/worker2 exit... wait process dmctl_basic/worker2 exit... process dmctl_basic/worker2 already exit dmctl test cmd: "query-status" got=2 expected=2 got=2 expected=2 [Sun May 5 20:48:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:48:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive config dmctl test cmd: "config" dmctl test cmd: "config hihi haha" dmctl test cmd: "config task haha" dmctl test cmd: "config master haha" dmctl test cmd: "config worker haha" dmctl test cmd: "config source haha" dmctl test cmd: "get-config haha" dmctl test cmd: "get-config haha hihi" dmctl test cmd: "get-config master haha" dmctl test cmd: "get-config worker haha" dmctl test cmd: "get-config source haha" dmctl test cmd: "get-config task haha" dmctl test cmd: "get-task-config haha" dmctl test cmd: "get-task-config haha hihi" dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "stop-task test" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 22.96 secs (127265974 bytes/sec) [Pipeline] { [Pipeline] cache dmctl test cmd: "operate-source stop mysql-replica-01" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source1.yaml" dmctl test cmd: "operate-source stop mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "start-task /tmp/dm_test/dmctl_basic/get_task.yaml" wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Sun May 5 20:48:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 11.33 secs (257871044 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:47 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 5 20:47 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:47 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 .github -rw-r--r--. 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:47 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:47 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 20:47 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:47 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 5 20:47 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:47 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:47 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 examples -rw-r--r--. 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 5 20:47 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:47 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:47 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:47 tests -rw-r--r--. 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:47 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 5 20:47 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:47 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 .github -rw-r--r--. 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:47 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:47 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 20:47 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:47 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:47 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 5 20:48 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:48 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:48 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 examples -rw-r--r--. 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 5 20:48 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:48 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:48 tests -rw-r--r--. 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process worker1 exit... [Pipeline] sh [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 5 20:48 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:48 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 .github -rw-r--r--. 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 20:48 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:48 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 5 20:48 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:48 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:48 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 examples -rw-r--r--. 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 5 20:48 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:48 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:48 tests -rw-r--r--. 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh wait process worker1 exit... + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 5 20:48 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:48 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 .github -rw-r--r--. 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:48 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 20:48 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:48 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 5 20:48 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:48 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:48 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 examples -rw-r--r--. 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:48 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 5 20:48 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:48 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:48 tests -rw-r--r--. 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:48 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] cache + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:48 . drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:48 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 5 20:48 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:48 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:48 .github -rw-r--r-- 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:48 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:48 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 5 20:48 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:48 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:48 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 5 20:48 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:48 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:48 engine -rwxr-xr-x 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:48 examples -rw-r--r-- 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:48 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 5 20:48 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:48 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:48 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:48 tests -rw-r--r-- 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:48 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... process worker1 already exit wait process worker2 exit... wait process worker2 exit... process worker2 already exit dmctl test cmd: "list-member --worker" got=2 expected=2 [Sun May 5 20:48:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:48:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" show_ddl_locks_no_locks dmctl test cmd: "shard-ddl-lock test" dmctl test cmd: "show-ddl-locks test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status" dmctl test cmd: "pause-task test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=2 expected=2 dmctl master_addr dmctl test cmd: "list-member" dmctl test cmd: "list-member" dmctl test cmd: "list-member" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 15.09 secs (193629082 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 . drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 5 20:48 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 .github -rw-r--r-- 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 5 20:49 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 5 20:49 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 engine -rwxr-xr-x 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 examples -rw-r--r-- 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 5 20:49 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 tests -rw-r--r-- 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] } [Pipeline] // timeout [Pipeline] sh [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] sh + '[' TLS_GROUP == G01 ']' + echo 'run G01 test' run G01 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ dmctl test cmd: "list-member" dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" + make dm_integration_test_in_group GROUP=G01 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] sh + '[' TLS_GROUP == G04 ']' + echo 'run G04 test' run G04 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh + make dm_integration_test_in_group GROUP=G04 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + '[' TLS_GROUP == G05 ']' + echo 'run G05 test' run G05 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] // timeout [Pipeline] sh + make dm_integration_test_in_group GROUP=G05 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt dmctl test cmd: "start-relay -s mysql-replica-02 worker2" + '[' TLS_GROUP == G00 ']' + echo 'run G00 test' run G00 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G01 [Pipeline] // timeout [Pipeline] sh + make dm_integration_test_in_group GROUP=G00 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Run cases: ha_cases3 ha_cases3_1 ha_master tracker_ignored_ddl validator_checkpoint incompatible_ddl_changes ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + '[' TLS_GROUP == G09 ']' + echo 'run G09 test' run G09 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G09 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G04 Run cases: import_goroutine_leak incremental_mode initial_unit drop_column_with_index duplicate_event expression_filter extend_column fake_rotate_event import_goroutine_leak not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] { [Pipeline] { /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G05 Run cases: load_interrupt many_tables online_ddl foreign_key full_mode gbk gtid ha_cases http_proxies load_interrupt not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G00 Run cases: ha_cases_1 ha_cases_2 ha_cases2 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { dmctl test cmd: "resume-task test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=4 expected=4 check diff successfully dmctl test cmd: "pause-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" + '[' TLS_GROUP == G07 ']' + echo 'run G07 test' run G07 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G09 Run cases: import_v10x sharding2 ha new_collation_off only_dml openapi s3_dumpling_lightning sequence_sharding_optimistic ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "resume-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" + make dm_integration_test_in_group GROUP=G07 [Pipeline] { check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server dmctl test cmd: "purge-relay --filename dm-it-aec7491a-d0de-4359-8cb8-49993d818fb5-bqn9t-2f8qw-bin.000003 -s mysql-replica-01" /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] { [Pipeline] { dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "encrypt abc" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:17 CST 2024] <<<<<< start test_stop_task >>>>>> [Sun May 5 20:49:17 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive encrypt value: Lpm2Nof8TKVLprJZx1uPbUDZ3Ww= 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G07 process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table Run cases: shardddl1 shardddl1_1 shardddl2 shardddl2_1 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [many_tables] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [incremental_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 5 12:49 . drwxr-sr-x 17 1000 1000 4.0K May 5 12:49 .. -rw------- 1 root 1000 1.7K May 5 12:49 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 ca.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 client-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 client-key.pem -rw------- 1 root 1000 1.7K May 5 12:49 private_key.pem -rw-r--r-- 1 root 1000 451 May 5 12:49 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 server-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 server-key.pem [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:18 CST 2024] <<<<<< start test_kill_master >>>>>> [Sun May 5 20:49:18 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 5 12:49 . drwxr-sr-x 17 1000 1000 4.0K May 5 12:49 .. -rw------- 1 root 1000 1.7K May 5 12:49 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 ca.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 client-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 client-key.pem -rw------- 1 root 1000 1.7K May 5 12:49 private_key.pem -rw-r--r-- 1 root 1000 451 May 5 12:49 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 server-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 server-key.pem rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:49:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] sh wait process dm-master.test exit... process dm-master.test already exit wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh import prepare data start DM worker and master [Sun May 5 20:49:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master1.toml >>>>>> [Sun May 5 20:49:19 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master2.toml >>>>>> [Sun May 5 20:49:19 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master3.toml >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 5 12:49 . drwxr-sr-x 17 1000 1000 4.0K May 5 12:49 .. -rw------- 1 root 1000 1.7K May 5 12:49 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 ca.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 client-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 client-key.pem -rw------- 1 root 1000 1.7K May 5 12:49 private_key.pem -rw-r--r-- 1 root 1000 451 May 5 12:49 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 server-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 server-key.pem wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [import_v10x] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit source1 master status, dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 1049, 96ae8793-0add-11ef-bf03-aa1057f7af0c:1-4 source2 master status, dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 1158, 9707fb65-0add-11ef-aee9-aa1057f7af0c:1-4 [Sun May 5 20:49:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time version() 5.7.26-log version() 8.0.21 start running case: [shardddl1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 5 12:49 . drwxr-sr-x 17 1000 1000 4.0K May 5 12:49 .. -rw------- 1 root 1000 1.7K May 5 12:49 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 ca.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 client-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 client-key.pem -rw------- 1 root 1000 1.7K May 5 12:49 private_key.pem -rw-r--r-- 1 root 1000 451 May 5 12:49 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 server-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 server-key.pem + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 5 12:49 . drwxr-sr-x. 17 1000 1000 4.0K May 5 12:49 .. -rw-------. 1 root 1000 1.7K May 5 12:49 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:49 ca.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:49 client-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:49 client-key.pem -rw-------. 1 root 1000 1.7K May 5 12:49 private_key.pem -rw-r--r--. 1 root 1000 451 May 5 12:49 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 5 12:49 server-cert.pem -rw-------. 1 root 1000 1.7K May 5 12:49 server-key.pem import prepare data start DM worker and master cluster [Sun May 5 20:49:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Sun May 5 20:49:20 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Sun May 5 20:49:20 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] } [Pipeline] } [Pipeline] } wait process dm-worker.test exit... [Pipeline] } [Pipeline] } rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:49:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // container [Pipeline] // container + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 5 12:49 . drwxr-sr-x 17 1000 1000 4.0K May 5 12:49 .. -rw------- 1 root 1000 1.7K May 5 12:49 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 ca.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 client-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 client-key.pem -rw------- 1 root 1000 1.7K May 5 12:49 private_key.pem -rw-r--r-- 1 root 1000 451 May 5 12:49 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 5 12:49 server-cert.pem -rw------- 1 root 1000 1.7K May 5 12:49 server-key.pem [Pipeline] // container rpc addr 127.0.0.1:8263 is alive [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] // container wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:49:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source1.yaml" [Sun May 5 20:49:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Sun May 5 20:49:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "config task test --path /tmp/dm_test/import_v10x/task.yaml" got=1 expected=1 start prepare_data rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source1.yaml" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source2.yaml" rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sun May 5 20:49:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "list-member" [Sun May 5 20:49:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:49:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "encrypt a" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait process dm-worker exit... wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source2.yaml" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... [Sun May 5 20:49:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:26 CST 2024] <<<<<< test case dmctl_basic success! >>>>>> start running case: [dmctl_command] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh... Verbose mode = false wait process dm-worker exit... process dm-worker already exit [Sun May 5 20:49:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sun May 5 20:49:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:49:26 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:26 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:49:27 CST 2024] <<<<<< start DM-COMPACT >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8264 is alive [Sun May 5 20:49:27 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-02 worker1" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " rpc addr 127.0.0.1:8261 is alive exit code should be not zero exit code should be not zero [Sun May 5 20:49:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-01 worker2" dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:18262 is alive [Sun May 5 20:49:28 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase wait process dm-worker.test exit... rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml " got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source1-encrypted-pass.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/wrong-source.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml -w worker1" dmctl test cmd: "list-member --name worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w wrong-worker" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:30 CST 2024] <<<<<< test case import_v10x success! >>>>>> start running case: [sharding2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w worker1" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:49:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source1.yaml" use sync_diff_inspector to check increment data check diff successfully [Sun May 5 20:49:32 CST 2024] <<<<<< finish test_running >>>>>> kill dm-master1 [Sun May 5 20:49:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source2.yaml" check diff successfully check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again wait process dm-master1 exit... wait process dm-master1 exit... process dm-master1 already exit waiting 5 seconds dmctl test cmd: "start-relay -s mysql-replica-02 worker2" finish prepare_data [Sun May 5 20:49:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "list-member" [Sun May 5 20:49:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 11.79 secs (247870700 bytes/sec) [Pipeline] { [Pipeline] cache got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time use sync_diff_inspector to check increment data check diff successfully rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... check diff successfully [Sun May 5 20:49:37 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" check diff failed 3-th time, retry later check sync diff for the increment replication check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:49:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/many_tables/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task.yaml " got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" check sync diff for the increment replication check diff failed 1-th time, retry later check diff failed 4-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "stop-relay -s mysql-replica-02 worker1" wait process dm-worker.test exit... got=2 expected=2 start dumping SQLs into source stop tasks test dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source1.yaml" got=3 expected=3 dmctl test cmd: "query-status test" dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "query-status test" got=1 expected=1 stop tasks test2 dmctl test cmd: "stop-task test2" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "start-relay -s mysql-replica-02 worker1" check task is running HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive check master2,3 are running dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=3 expected=3 dmctl test cmd: "query-status test2" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-01 worker2" got=2 expected=2 start task in incremental mode with zero gtid/pos [Sun May 5 20:49:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... got=1 expected=1 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully kill dm-worker1 dmctl test cmd: "list-member --worker" check diff failed 5-th time, retry later got=2 expected=2 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 4.61 secs (633434384 bytes/sec) [Pipeline] { [Pipeline] cache wait process dm-worker1 exit... use sync_diff_inspector to check increment2 data now! check diff successfully [Sun May 5 20:49:42 CST 2024] <<<<<< finish test_kill_master >>>>>> [Sun May 5 20:49:42 CST 2024] <<<<<< start test_kill_and_isolate_worker >>>>>> [Sun May 5 20:49:42 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:49:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=4 expected=4 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml" wait process dm-master.test exit... wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 check diff failed 6-th time, retry later rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker2 exit... dmctl test cmd: "query-status test2" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-master.test exit... got=4 expected=4 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait process dm-worker.test exit... wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:49:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 7-th time, retry later 0 in Load unit "progress": "0.00 %", wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:49:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive wait for port 8262 offline the 1-th time dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:49:48 CST 2024] <<<<<< finish DM-COMPACT >>>>>> rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... wait for port 8262 offline the 2-th time wait process dm-worker exit... check diff successfully dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... [Sun May 5 20:49:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker exit... process dm-worker already exit [Sun May 5 20:49:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sun May 5 20:49:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:49:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time 0 in Load unit "progress": "0.00 %", rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:49:51 CST 2024] <<<<<< start DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again [2024/05/05 20:49:43.045 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=94f2b32d-0add-11ef-81ad-e61fb9ae307f.000001/dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14-bin.000001] [2024/05/05 20:49:43.060 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=94f2b32d-0add-11ef-81ad-e61fb9ae307f.000001/dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14-bin.000001] [2024/05/05 20:49:46.601 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=94f2b32d-0add-11ef-81ad-e61fb9ae307f.000001/dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14-bin.000001] [2024/05/05 20:49:46.618 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=94f2b32d-0add-11ef-81ad-e61fb9ae307f.000001/dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14-bin.000001] [2024/05/05 20:49:46.635 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=94f2b32d-0add-11ef-81ad-e61fb9ae307f.000001/dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14-bin.000001] [2024/05/05 20:49:46.655 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=94f2b32d-0add-11ef-81ad-e61fb9ae307f.000001/dm-it-e7a7380c-c6d6-4857-b711-11aa9f2660f7-w5987-86f14-bin.000001] dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "pause-task test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" check diff successfully check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=2 check sync diff for the increment replication check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 import prepare data start DM worker and master cluster [Sun May 5 20:49:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Sun May 5 20:49:53 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Sun May 5 20:49:53 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=3 expected=3 got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 11.04 secs (264711890 bytes/sec) [Pipeline] { [Pipeline] cache run tidb sql failed 2-th time, retry later dmctl test cmd: "validation status test" got=4 expected=4 got=1 expected=1 dmctl test cmd: "validation status --table-stage running test" got=4 expected=4 got=0 expected=0 dmctl test cmd: "validation status --table-stage stopped test" got=2 expected=2 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=0 expected=1 command: validation show-error --error all test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... process dm-master.test already exit 0 in Load unit "progress": "28.64 %", wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time run tidb sql failed 3-th time, retry later rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sun May 5 20:49:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status --table-stage stopped" dmctl test cmd: "validation status --table-stage start test" dmctl test cmd: "validation show-error --error resolved test" dmctl test cmd: "validation show-error --error all" dmctl test cmd: "validation ignore-error test 100 --all" dmctl test cmd: "validation resolve-error test 100 --all" dmctl test cmd: "validation clear-error test 100 --all" dmctl test cmd: "validation ignore-error test 100 101" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:49:58 CST 2024] <<<<<< test case sharding2 success! >>>>>> start running case: [ha] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM worker and master [Sun May 5 20:49:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master1.toml >>>>>> [Sun May 5 20:49:58 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "validation resolve-error test 100 101" dmctl test cmd: "validation clear-error test 100 101" dmctl test cmd: "validation ignore-error test error-id" dmctl test cmd: "validation resolve-error test error-id" run tidb sql failed 4-th time, retry later dmctl test cmd: "validation clear-error test error-id" dmctl test cmd: "validation ignore-error test" dmctl test cmd: "validation resolve-error test" dmctl test cmd: "validation clear-error test" dmctl test cmd: "validation ignore-error" dmctl test cmd: "validation resolve-error" [Sun May 5 20:49:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "validation clear-error" dmctl test cmd: "validation ignore-error non-exist-task-name 1" dmctl test cmd: "validation resolve-error non-exist-task-name 1" dmctl test cmd: "validation clear-error non-exist-task-name 1" dmctl test cmd: "validation resolve-error test 1" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation show-error --error unprocessed test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" run tidb sql failed 5-th time, retry later got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test 2" dmctl test cmd: "validation status test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" got=1 expected=1 dmctl test cmd: "validation clear-error test 1" 0 in Load unit "progress": "56.70 %", dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" got=1 expected=1 dmctl test cmd: "validation clear-error test 2" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation stop test" got=1 expected=1 dmctl test cmd: "validation status test" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=3 expected=3 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 59259 0 59259 0 0 22.9M 0 --:--:-- --:--:-- --:--:-- 28.2M % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 67318 0 67318 0 0 25.5M 0 --:--:-- --:--:-- --:--:-- 32.0M dmctl test cmd: "validation show-error --error all test" got=1 expected=1 rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Sun May 5 20:50:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "validation ignore-error test 3" dmctl test cmd: "validation resolve-error test 4" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation clear-error test --all" dmctl test cmd: "validation status test" run tidb sql failed 6-th time, retry later got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 6.07 secs (481310069 bytes/sec) [Pipeline] { got=2 expected=2 got=2 expected=2 dmctl test cmd: "validation status test" got=2 expected=2 dmctl test cmd: "query-status test" [Pipeline] } [Pipeline] cache rpc addr 127.0.0.1:8262 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha/source1.yaml" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "stop-task test" clean up data dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/dm-task.yaml --remove-meta" dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Sun May 5 20:50:04 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "validation status test" got=2 expected=2 --> validation resolve-error --all 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive run tidb sql failed 7-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha/source2.yaml" wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:50:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... 0 in Load unit "progress": "83.16 %", use sync_diff_inspector to check increment data check diff successfully [Sun May 5 20:50:06 CST 2024] <<<<<< finish test_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:50:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-02 worker2" run tidb sql failed 8-th time, retry later wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive got=2 expected=2 kill dm-worker2 got=2 expected=2 [Sun May 5 20:50:08 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker2 exit... run tidb sql failed 9-th time, retry later rpc addr 127.0.0.1:8461 is alive ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 1-th time, got value: '' wait process dm-worker.test exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status test" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 4.18 secs (698738857 bytes/sec) [Pipeline] { got=1 expected=1 [Sun May 5 20:50:10 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] sh ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 2-th time, got value: '' + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 . drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 5 20:49 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 .github -rw-r--r-- 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 5 20:49 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 5 20:49 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 engine -rwxr-xr-x 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 examples -rw-r--r-- 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 5 20:49 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 tests -rw-r--r-- 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh rpc addr 127.0.0.1:8264 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Sun May 5 20:50:11 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time [Pipeline] sh run tidb sql failed 10-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 . drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 5 20:49 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 .github -rw-r--r-- 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 5 20:49 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 5 20:49 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 engine -rwxr-xr-x 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 examples -rw-r--r-- 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 5 20:49 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 tests -rw-r--r-- 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 3-th time, got value: '' rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 . drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 5 20:49 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 .github -rw-r--r-- 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:50 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 5 20:50 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:50 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:50 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 5 20:50 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:50 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:50 engine -rwxr-xr-x 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:50 examples -rw-r--r-- 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:50 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 5 20:50 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:50 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:50 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:50 tests -rw-r--r-- 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:50 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break 0 in Sync unit + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:50 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:50 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 5 20:50 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:50 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:50 .github -rw-r--r--. 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:50 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 20:50 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 20:50 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:50 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:50 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 5 20:50 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 5 20:50 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 5 20:50 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:50 examples -rw-r--r--. 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 20:50 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 5 20:50 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:50 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:50 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 5 20:50 tests -rw-r--r--. 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 20:50 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3 worker4" metric dm_master_worker_state{worker="worker1"} has invalid value start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml " rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:50:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 . drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 5 20:49 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 .github -rw-r--r-- 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:49 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 5 20:49 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 5 20:49 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:49 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:49 engine -rwxr-xr-x 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 examples -rw-r--r-- 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:49 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 5 20:49 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:49 tests -rw-r--r-- 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:49 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" run tidb sql failed 11-th time, retry later got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=3 expected=3 restart dm-worker3 got=1 expected=1 dmctl test cmd: "resume-task test" got=3 expected=3 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data check diff failed 1-th time, retry later wait process dm-worker3 exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" wait process dm-worker3 exit... process dm-worker3 already exit wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Sun May 5 20:50:15 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time run tidb sql failed 12-th time, retry later check diff successfully dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8264 is alive isolate dm-worker4 kill dm-worker4 check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" start dm-worker3 and kill dm-worker2 wait process dm-worker4 exit... run tidb sql failed 13-th time, retry later wait process dm-worker2 exit... check diff successfully dmctl test cmd: "validation status test" [Pipeline] // container got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 3648 0 3648 0 0 506k 0 --:--:-- --:--:-- --:--:-- 593k dmctl test cmd: "stop-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "stop-relay -s mysql-replica-02 worker1" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" got=2 expected=2 dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:50:18 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache wait process dm-worker4 exit... process dm-worker4 already exit [Sun May 5 20:50:18 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 isolate dm-worker3 kill dm-worker3 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive start incremental_data run tidb sql failed 14-th time, retry later rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-worker3 exit... got=2 expected=2 wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again wait process dm-worker3 exit... process dm-worker3 already exit [Sun May 5 20:50:20 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 disable isolate dm-worker4 kill dm-worker4 run tidb sql failed 15-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker4 exit... wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process dm-worker4 exit... process dm-worker4 already exit [Sun May 5 20:50:22 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=3 command: query-status test "stage": "Running" count: 1 != expected: 3, failed the 0-th time, will retry again wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:50:23 CST 2024] <<<<<< finish DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "validation show-error test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:23 CST 2024] <<<<<< test case incremental_mode success! >>>>>> start running case: [initial_unit] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again failpoint=github.com/pingcap/tiflow/dm/syncer/LoadCheckpointFailed=return(1152) [Sun May 5 20:50:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker exit... got=3 expected=3 query-status from all dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "pause-task test" wait process dm-worker exit... process dm-worker already exit [Sun May 5 20:50:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sun May 5 20:50:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time restart worker4 wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation resolve-error test --all" dmctl test cmd: "validation show-error test" got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 --> validation ignore-error --all 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker4 exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:50:26 CST 2024] <<<<<< start DM-MULTIPLE_ROWS >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker4 exit... process dm-worker4 already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/0/source1.yaml" wait for the task to be scheduled and keep paused dmctl test cmd: "query-status test" [Sun May 5 20:50:27 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive resume task before kill and restart dm-worker dmctl test cmd: "resume-task test" start dm-worker2 and kill dm-worker3 got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "resume-task test" dmctl test cmd: "pause-task test" wait process dm-worker3 exit... wait process dm-worker.test exit... check log contain failed 2-th time, retry later wait process dm-worker3 exit... process dm-worker3 already exit [Sun May 5 20:50:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=2 expected=2 finish incremental_data check diff 1 dmctl test cmd: "query-status test" resume task will also initial failed dmctl test cmd: "resume-task test" use sync_diff_inspector to check increment2 data now! check diff successfully [Sun May 5 20:50:31 CST 2024] <<<<<< finish test_kill_and_isolate_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... check diff failed 1-th time, retry later dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:50:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm) 2921923584 bytes in 14.14 secs (206655581 bytes/sec) [Pipeline] { wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" run tidb sql failed 2-th time, retry later [Pipeline] } [Pipeline] } + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 . drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 5 20:43 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 5 20:50 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 5 20:43 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:50 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:50 .github -rw-r--r-- 1 jenkins jenkins 985 May 5 20:43 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 5 20:43 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 5 20:43 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 5 20:43 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 5 20:43 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 5 20:43 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 5 20:43 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 5 20:43 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 5 20:43 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 5 20:43 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 5 20:43 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 5 20:43 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:50 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 5 20:50 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 5 20:50 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:50 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:50 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 5 20:50 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 5 20:50 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 5 20:50 engine -rwxr-xr-x 1 jenkins jenkins 27K May 5 20:43 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:50 examples -rw-r--r-- 1 jenkins jenkins 21K May 5 20:43 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 5 20:43 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 5 20:50 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 5 20:50 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:50 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:50 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 5 20:50 tests -rw-r--r-- 1 jenkins jenkins 582 May 5 20:43 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 5 20:50 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:50:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" [Pipeline] } [Pipeline] } [Pipeline] } rpc addr 127.0.0.1:8262 is alive check diff successfully [2024/05/05 20:49:39.250 +08:00] [INFO] [dumpling.go:111] ["create dumpling"] [task=test] [unit=dump] [config="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"auto\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"\",\"ServerInfo\":{\"ServerType\":0,\"ServerVersion\":null,\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/05 20:49:39.378 +08:00] [INFO] [dump.go:151] ["begin to run Dump"] [task=test] [unit=dump] [conf="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"flush\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"sql\",\"ServerInfo\":{\"ServerType\":1,\"ServerVersion\":\"5.7.26-log\",\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/05 20:49:49.273 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14321] [2024/05/05 20:49:59.274 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14321] [2024/05/05 20:50:09.275 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14321] [2024/05/05 20:50:19.276 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=9474220] [2024/05/05 20:50:29.276 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=15530517] check diff 2 run tidb sql failed 3-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... [Pipeline] // timeout [Pipeline] // timeout [Pipeline] sh [Pipeline] sh check diff successfully run tidb sql failed 4-th time, retry later + '[' TLS_GROUP == TLS_GROUP ']' + echo 'run tls test' run tls test + echo 'copy mysql certs' copy mysql certs + sudo mkdir -p /var/lib/mysql + sudo chmod 777 /var/lib/mysql + sudo chown -R 1000:1000 /var/lib/mysql + sudo cp -r /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ca-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ca.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/client-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/client-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/private_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/public_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/server-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/server-key.pem /var/lib/mysql/ + sudo chown -R 1000:1000 /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem + ls -alh /var/lib/mysql/ total 40K drwxrwxrwx 2 jenkins jenkins 4.0K May 5 20:50 . drwxr-xr-x 1 root root 4.0K May 5 20:50 .. -rw------- 1 jenkins jenkins 1.7K May 5 20:50 ca-key.pem -rw-r--r-- 1 jenkins jenkins 1.1K May 5 20:50 ca.pem -rw-r--r-- 1 jenkins jenkins 1.1K May 5 20:50 client-cert.pem -rw------- 1 jenkins jenkins 1.7K May 5 20:50 client-key.pem -rw------- 1 jenkins jenkins 1.7K May 5 20:50 private_key.pem -rw-r--r-- 1 jenkins jenkins 451 May 5 20:50 public_key.pem -rw-r--r-- 1 jenkins jenkins 1.1K May 5 20:50 server-cert.pem -rw------- 1 jenkins jenkins 1.7K May 5 20:50 server-key.pem + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] } check diff successfully dmctl test cmd: "validation status test" wait process dm-master.test exit... wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Sun May 5 20:50:39 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time [Pipeline] // timeout [Pipeline] // timeout + make dm_integration_test_in_group GROUP=TLS_GROUP /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] // timeout got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again [Pipeline] sh wait process dm-worker.test exit... [Pipeline] sh wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:40 CST 2024] <<<<<< test case ha_cases_1 success! >>>>>> start running case: [ha_cases_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:40 CST 2024] <<<<<< start test_kill_master_in_sync >>>>>> [Sun May 5 20:50:40 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive [Pipeline] sh + '[' TLS_GROUP == G11 ']' + echo 'run G11 test' run G11 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G11 process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table [Pipeline] // timeout rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker3" dmctl test cmd: "list-member --name worker3" /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] sh + '[' TLS_GROUP == G02 ']' + echo 'run G02 test' run G02 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + '[' TLS_GROUP == G08 ']' + echo 'run G08 test' run G08 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:50:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 5-th time, retry later got=1 expected=1 query-status from all dm-master dmctl test cmd: "query-status test" + make dm_integration_test_in_group GROUP=G02 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + make dm_integration_test_in_group GROUP=G08 + '[' TLS_GROUP == G10 ']' + echo 'run G10 test' run G10 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G10 got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" + '[' TLS_GROUP == G06 ']' + echo 'run G06 test' run G06 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again got=3 expected=3 join new dm-master and query-status [Sun May 5 20:50:41 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time + make dm_integration_test_in_group GROUP=G06 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G11 Run cases: validator_basic dm_syncer shardddl_optimistic slow_relay_writer sql_mode sync_collation ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G02 Run cases: handle_error handle_error_2 handle_error_3 adjust_gtid async_checkpoint_flush binlog_parse case_sensitive checkpoint_transaction check_task ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G08 Run cases: shardddl3 shardddl3_1 shardddl4 shardddl4_1 sharding sequence_sharding sequence_sharding_removemeta ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) import prepare data start DM worker and master cluster [Sun May 5 20:50:42 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Sun May 5 20:50:42 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Sun May 5 20:50:42 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh TLS_GROUP Run cases: tls ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G10 Run cases: start_task print_status http_apis new_relay all_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G06 Run cases: relay_interrupt safe_mode sequence_safe_mode lightning_load_task lightning_mode metrics ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run tidb sql failed 6-th time, retry later rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "query-status test" got=3 expected=3 rpc addr 127.0.0.1:8262 is alive got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [shardddl3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh... Verbose mode = false VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [validator_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh... Verbose mode = false --> full mode, check we validate different data types(gtid=false, relay=false) 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [handle_error] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [start_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh... Verbose mode = false start running case: [tls] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [relay_interrupt] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetGlobalVariableFailed=return("server_uuid,1152") dmctl test cmd: "validation show-error test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully 39 11 11 66 dmctl test cmd: "stop-task test" [Sun May 5 20:50:45 CST 2024] <<<<<< finish DM-MULTIPLE_ROWS >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:50:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sun May 5 20:50:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source1.yaml" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:50:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source1.yaml" [Sun May 5 20:50:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/source1.yaml" dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker exit... process dm-worker already exit [Sun May 5 20:50:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:50:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> [Sun May 5 20:50:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source2.yaml" wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/0/source1.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source2.yaml" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test --all" dmctl test cmd: "validation show-error test" got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sun May 5 20:50:47 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:50:48 CST 2024] <<<<<< start DM-CAUSALITY >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" [Sun May 5 20:50:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 [Sun May 5 20:50:48 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" [Sun May 5 20:50:48 CST 2024] <<<<<< start DM-071 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status test" wait process tidb-server exit... got=3 expected=3 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" waiting for asynchronous relay and subtask to be started wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again [Sun May 5 20:50:50 CST 2024] <<<<<< finish DM-CAUSALITY >>>>>> [Sun May 5 20:50:50 CST 2024] <<<<<< start DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb2 add column c int default 100; alter table handle_error.tb2 add primary key (c);" check diff successfully dmctl test cmd: "stop-task test" wait process tidb-server exit... process tidb-server already exit start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog inject test -s mysql-replica-01 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique(c);" [Sun May 5 20:50:50 CST 2024] <<<<<< finish DM-071 pessimistic >>>>>> [Sun May 5 20:50:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Sun May 5 20:50:50 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Sun May 5 20:50:50 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "binlog inject test -s mysql-replica-02 alter table handle_error.tb2 drop primary key; alter table handle_error.tb2 add unique(c);" run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-worker.test exit... dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-05 14:50:49'" wait process dm-worker.test exit... dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status" dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later reset go failpoints, and need restart dm-worker then resume task, task will recover success dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase [Sun May 5 20:50:51 CST 2024] <<<<<< start DM-073 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "stop-task test" dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=false, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sun May 5 20:50:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time failpoint=github.com/pingcap/tiflow/dm/syncer/dbconn/GetMasterStatusFailed=return(1152) [Sun May 5 20:50:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "stop-task test" [Sun May 5 20:50:53 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR >>>>>> [Sun May 5 20:50:53 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:50:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker2" [Sun May 5 20:50:53 CST 2024] <<<<<< finish DM-073 pessimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-05 14:50:49'" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:50:54 CST 2024] <<<<<< finish DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> [Sun May 5 20:50:54 CST 2024] <<<<<< start DM-UpdateBARule optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:50:54 CST 2024] <<<<<< START DM-MASTER on port 8761, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master6.toml >>>>>> wait for rpc addr 127.0.0.1:8761 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:50:54 CST 2024] <<<<<< start DM-076 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Sun May 5 20:50:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/1/source1.yaml" use sync_diff_inspector to check increment data check diff successfully [Sun May 5 20:50:55 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:50:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8761 is alive dmctl test cmd: "query-status test" wait process dm-worker.test exit... dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '1995-03-07 01:02:03'" got=2 expected=2 got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/shardddl1/task.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source1.yaml" [Sun May 5 20:50:56 CST 2024] <<<<<< finish DM-076 pessimistic >>>>>> [Sun May 5 20:50:56 CST 2024] <<<<<< start DM-076 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source2.yaml" wait process dm-master1 exit... dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2037-12-12 01:02:03'" check master alive dmctl test cmd: "list-member" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:50:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" dmctl test cmd: "show-ddl-locks" wait process dm-master1 exit... process dm-master1 already exit wait and check task running got=1 expected=1 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" check diff failed 1-th time, retry later got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" resume task will also initial failed dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:50:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 --> (fail) validation start: missing mode value dmctl test cmd: "validation start --mode" --> (fail) validation start: invalid mode dmctl test cmd: "validation start --mode xxx" dmctl test cmd: "query-status test" test http and api interface Release Version: v8.2.0-alpha-55-g76ad8e6db Git Commit Hash: 76ad8e6dbaeacfb7ca050ed5db5eba0401b203cf Git Branch: HEAD UTC Build Time: 2024-05-05 09:18:21 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true rpc addr 127.0.0.1:8261 is alive { "result": true, "msg": "", "members": [ { "leader": { "msg": "", "name": "master1", "addr": "127.0.0.1:8261" } }, { "master": { "msg": "", "masters": [ { "name": "master1", "memberID": "10068700568285020701", "alive": true, "peerURLs": [ "https://127.0.0.1:8291" ], "clientURLs": [ "https://127.0.0.1:8261" ] }, { "name": "master2", "memberID": "518295880577331676", "alive": true, "peerURLs": [ "https://127.0.0.1:8292" ], "clientURLs": [ "https://127.0.0.1:8361" ] }, { "name": "master3", "memberID": "6800138986734450252", "alive": true, "peerURLs": [ "https://127.0.0.1:8293" ], "clientURLs": [ "https://127.0.0.1:8461" ] } ] } }, { "worker": { "msg": "", "workers": [ { "name": "worker1", "addr": "127.0.0.1:8262", "stage": "bound", "source": "mysql-replica-01" } ] } } ] } rpc addr 127.0.0.1:8261 is alive use common name not in 'cert-allowed-cn' should not request success wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... --> (fail) validation start: missing start-time value dmctl test cmd: "validation start --start-time" --> (fail) validation start: invalid start-time dmctl test cmd: "validation start --start-time xx" --> (fail) validation start: without all-task and task-name restart dm-master rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" dmctl test cmd: "validation start --mode full" --> (fail) validation start: with both all-task and task-name dmctl test cmd: "validation start --all-task test" --> (fail) validation start: too many arguments dmctl test cmd: "validation start test test2" [Sun May 5 20:50:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time --> (fail) validation start: non-existed subtask dmctl test cmd: "validation start xxxxx" --> (fail) validation start: non-exist source dmctl test cmd: "validation start -s xxxxx --all-task" --> (fail) validation stop: without all-task and task-name dmctl test cmd: "validation stop" --> (fail) validation stop: with both all-task and task-name dmctl test cmd: "validation stop --all-task test" wait process dm-master.test exit... process dm-master.test already exit --> (fail) validation stop: too many arguments dmctl test cmd: "validation stop test test2" --> (fail) validation stop: non-existed subtask dmctl test cmd: "validation stop xxxxx" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:00 CST 2024] <<<<<< finish DM-076 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time --> (fail) validation stop: non-exist source dmctl test cmd: "validation stop -s xxxxx --all-task" --> (fail) validation stop: stop not-enabled validator dmctl test cmd: "validation stop -s mysql-replica-01 test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-master exit... --> (success) validation start: start validation without explicit mode for source 1 dmctl test cmd: "validation start -s mysql-replica-01 test" --> (fail) validation start: start all validator with explicit mode, but 1 subtask already enabled dmctl test cmd: "validation start --mode full test" kill dm-master1 dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success --> (fail) validation start: start validation with explicit mode for source 1 again dmctl test cmd: "validation start -s mysql-replica-01 --mode full test" --> (fail) validation start: start all validator without explicit mode dmctl test cmd: "validation start test" --> (fail) validation stop: stop all but 1 subtask is not enabled dmctl test cmd: "validation stop test" --> (success) validation start: start validation with fast mode and start-time for source 2 dmctl test cmd: "validation start -s mysql-replica-02 --mode fast --start-time '2020-01-01 00:01:00' test" [Sun May 5 20:51:01 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 3-th time wait process dm-worker.test exit... dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation start: start all validator of the task without explicit param again, i.e. resuming dmctl test cmd: "validation start test" wait process dm-master exit... process dm-master already exit wait process dm-master1 exit... wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 4-th time dmctl test cmd: "config source mysql-replica-01" wait process dm-worker.test exit... dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation stop: stop validation of source 1 dmctl test cmd: "validation stop -s mysql-replica-01 test" dmctl test cmd: "validation status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master1 exit... process dm-master1 already exit kill dm-master2 got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:51:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> (success) validation stop: stop all of test dmctl test cmd: "validation stop test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation stop: stop all of test again dmctl test cmd: "validation stop test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: start all of test dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation stop: multiple source dmctl test cmd: "validation stop -s mysql-replica-01 -s mysql-replica-02 test" dmctl test cmd: "validation status test" [Sun May 5 20:51:03 CST 2024] <<<<<< finish DM-077 >>>>>> got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: multiple source dmctl test cmd: "validation start -s mysql-replica-01 -s mysql-replica-02 test" dmctl test cmd: "validation status test" [Sun May 5 20:51:03 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" [Sun May 5 20:51:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 5-th time got=1 expected=1 check diff successfully read binlog from relay log failed, and will use remote binlog wait process dm-master2 exit... rpc addr 127.0.0.1:8262 is alive got=1 expected=1 got=1 expected=1 wait for rpc addr 127.0.0.1:8261 alive the 6-th time got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master2 exit... process dm-master2 already exit initial cluster of dm-masters have been killed now we will check whether joined masters can work normally dmctl test cmd: "stop-task test" [Sun May 5 20:51:04 CST 2024] <<<<<< finish DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2-no-validator.yaml --remove-meta" wait process dm-worker.test exit... [Sun May 5 20:51:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 7-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:51:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "show-ddl-locks" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:51:05 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=1 expected=1 run tidb sql failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 8-th time wait process dm-master.test exit... process dm-master.test already exit got=1 expected=1 --> (fail) validation update: on non-existed validator dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid 93d0bc9b-0add-11ef-9598-52bb23923c15:1-999" --> (success) validation start: start all tasks dmctl test cmd: "validation start --all-task --mode fast" --> (fail) validation update: missing task-name dmctl test cmd: "validation update" --> (fail) validation update: multi tasks dmctl test cmd: "validation update test test2" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/source1.yaml" --> (fail) validation update: non-existed subtask dmctl test cmd: "validation update xxxxx" --> (fail) validation update: non-exist source dmctl test cmd: "validation update -s xxxxx test" dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml --remove-meta" --> (fail) validation update: not specify cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-02" --> (fail) validation update: not specify cutover-binlog-pos dmctl test cmd: "validation update test2 -s mysql-replica-01" got=1 expected=1 check diff failed 1-th time, retry later --> (fail) validation update: invalid cutover-binlog-pos dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos xxx" --> (fail) validation update: specify both cutover-binlog-pos and cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos '(mysql-bin.000001, 2345)' --cutover-binlog-gtid 93d0bc9b-0add-11ef-9598-52bb23923c15:1-999" wait for rpc addr 127.0.0.1:8261 alive the 9-th time wait process dm-worker.test exit... --> (success) validation update: on exist source dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid 93d0bc9b-0add-11ef-9598-52bb23923c15:1-999" dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=0 got=2 expected=2 got=2 expected=2 got=0 expected=0 got=2 expected=2 got=1 expected=1 dmctl test cmd: "validation status test2" got=1 expected=1 got=0 expected=0 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 --> (success) validation stop: stop all tasks dmctl test cmd: "validation stop --all-task" dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=0 got=2 expected=2 got=0 expected=0 got=2 expected=2 dmctl test cmd: "validation status test2" got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 10-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "show-ddl-locks" got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:51:08 CST 2024] <<<<<< finish DM-UpdateBARule optimistic >>>>>> [Sun May 5 20:51:08 CST 2024] <<<<<< start DM-RENAME_TABLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 11-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:09 CST 2024] <<<<<< finish DM-078 >>>>>> [Sun May 5 20:51:09 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" command: query-status test task test has no source or not exist count: 0 != expected: 1 { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-01", "worker": "worker3", "result": null, "relayStatus": null }, "subTaskStatus": [ { "name": "test", "stage": "Running", "unit": "Sync", "result": null, "unresolvedDDLLockID": "", "sync": { "totalEvents": "1", "totalTps": "0", "recentTps": "0", "masterBinlog": "(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000002, 5693)", "masterBinlogGtid": "96ae8793-0add-11ef-bf03-aa1057f7af0c:1-24", "syncerBinlog": "(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000002, 5693)", "syncerBinlogGtid": "96ae8793-0add-11ef-bf03-aa1057f7af0c:1-24", "blockingDDLs": [ ], "unresolvedGroups": [ ], "synced": true, "binlogType": "remote", "secondsBehindMaster": "0", "blockDDLOwner": "", "conflictMsg": "", "totalRows": "1", "totalRps": "0", "recentRps": "0" }, "validation": null } ] }, { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-02", "worker": "worker2", "result": null, "relayStatus": { "masterBinlog": "(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000002, 487)", "masterBinlogGtid": "9707fb65-0add-11ef-aee9-aa1057f7af0c:1-5", "relaySubDir": "9707fb65-0add-11ef-aee9-aa1057f7af0c.000001", "relayBinlog": "(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000002, 487)", "relayBinlogGtid": "9707fb65-0add-11ef-aee9-aa1057f7af0c:1-5", "relayCatchUpMaster": true, "stage": "Running", "result": null } }, "subTaskStatus": [ { "name": "test", "stage": "Running", "unit": "Sync", "result": null, "unresolvedDDLLockID": "", "sync": { "totalEvents": "1", "totalTps": "0", "recentTps": "0", "masterBinlog": "(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000002, 487)", "masterBinlogGtid": "9707fb65-0add-11ef-aee9-aa1057f7af0c:1-5", "syncerBinlog": "(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin|000001.000002, 487)", "syncerBinlogGtid": "9707fb65-0add-11ef-aee9-aa1057f7af0c:1-5", "blockingDDLs": [ ], "unresolvedGroups": [ ], "synced": true, "binlogType": "local", "secondsBehindMaster": "0", "blockDDLOwner": "", "conflictMsg": "", "totalRows": "1", "totalRps": "0", "recentRps": "0" }, "validation": null } ] } ] } PASS coverage: 3.3% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 12-th time check diff failed 1-th time, retry later dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 13-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:51:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 14-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:13 CST 2024] <<<<<< finish DM-RENAME_TABLE pessimistic >>>>>> [Sun May 5 20:51:13 CST 2024] <<<<<< start DM-RENAME_TABLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 15-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:13 CST 2024] <<<<<< finish DM-078 >>>>>> dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:51:14 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Sun May 5 20:51:15 CST 2024] <<<<<< finish DM-RENAME_TABLE optimistic >>>>>> [Sun May 5 20:51:15 CST 2024] <<<<<< start DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" use sync_diff_inspector to check increment2 data now! check diff successfully 4 dm-master alive 3 dm-worker alive 0 dm-syncer alive check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive wait for rpc addr 127.0.0.1:8261 alive the 16-th time wait process dm-master.test exit... [Sun May 5 20:51:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" check diff successfully dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "stop-task test" wait process dm-master.test exit... dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 17-th time [Sun May 5 20:51:16 CST 2024] <<<<<< finish DM-079 >>>>>> [Sun May 5 20:51:16 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 18-th time wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" [Sun May 5 20:51:18 CST 2024] <<<<<< finish DM-079 >>>>>> wait process dm-worker.test exit... dmctl test cmd: "check-task /tmp/dm_test/dmctl_command/temp.yaml" wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 19-th time 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Sun May 5 20:51:18 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> [Sun May 5 20:51:18 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:51:19 CST 2024] <<<<<< start DM-080 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 20-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchTargetDoTablesFailed=return(1152) wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:19 CST 2024] <<<<<< test case initial_unit success! >>>>>> start running case: [drop_column_with_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetSessionVariableFailed=return("sql_mode,1152") wait process dm-master.test exit... check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 check alive timeout check data check diff successfully ============================== test_master_ha_when_enable_tidb_and_only_ca_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Sun May 5 20:51:20 CST 2024] <<<<<< finish DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:51:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... [Sun May 5 20:51:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... use sync_diff_inspector to check increment data check diff successfully check diff successfully [Sun May 5 20:51:06 CST 2024] <<<<<< finish test_stop_task >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:15 CST 2024] <<<<<< test case ha_cases3 success! >>>>>> start running case: [ha_cases3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:16 CST 2024] <<<<<< start test_isolate_master_and_worker >>>>>> [Sun May 5 20:51:16 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master [Sun May 5 20:51:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> [Sun May 5 20:51:18 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> [Sun May 5 20:51:18 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-master exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... got=4 expected=4 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Sun May 5 20:51:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:51:22 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> [Sun May 5 20:51:22 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master exit... process dm-master already exit [Sun May 5 20:51:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/0/source1.yaml" wait process dm-master.test exit... check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/drop_column_with_index/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-task.yaml --remove-meta" start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "stop-task test" [Sun May 5 20:51:23 CST 2024] <<<<<< finish DM-080 pessimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:23 CST 2024] <<<<<< test case dmctl_command success! >>>>>> start running case: [downstream_diff_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:51:24 CST 2024] <<<<<< start DM-081 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:24 CST 2024] <<<<<< test case ha success! >>>>>> start running case: [new_collation_off] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01,mysql-replica-02 alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" dmctl test cmd: "query-status test" [Sun May 5 20:51:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:51:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... got=4 expected=4 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/1/source1.yaml" check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" wait process dm-master.test exit... query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" got=2 expected=2 [Sun May 5 20:51:25 CST 2024] <<<<<< start DM-RemoveLock pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "stop-task test" waiting for asynchronous relay and subtask to be started wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" [Sun May 5 20:51:26 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again [Sun May 5 20:51:26 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> [Sun May 5 20:51:26 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:51:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:51:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8264 is alive [Sun May 5 20:51:27 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:28 CST 2024] <<<<<< finish DM-081 pessimistic >>>>>> dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "query-status" reset go failpoints, and need restart dm-worker then resume task, task will recover success dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source2.yaml" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" check log contain failed 1-th time, retry later rpc addr 127.0.0.1:18262 is alive [Sun May 5 20:51:28 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time check log contain failed 1-th time, retry later [Sun May 5 20:51:29 CST 2024] <<<<<< start DM-082 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 97k 0 --:--:-- --:--:-- --:--:-- 104k got=1 expected=1 got=1 expected=1 got=1 expected=1 --> fast mode, check we validate different data types 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sun May 5 20:51:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task2.yaml " wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-worker1.toml >>>>>> dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:51:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" run tidb sql failed 1-th time, retry later [Sun May 5 20:51:31 CST 2024] <<<<<< finish DM-RemoveLock pessimistic >>>>>> [Sun May 5 20:51:31 CST 2024] <<<<<< start DM-RemoveLock optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/source2.yaml" dmctl test cmd: "query-status test" got=4 expected=4 got=2 expected=2 dmctl test cmd: "query-status test2" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Sun May 5 20:51:31 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> [Sun May 5 20:51:31 CST 2024] <<<<<< start DM-LIST_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-task.yaml" got=4 expected=4 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 82256 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused rpc addr 127.0.0.1:8262 is alive check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase rpc addr 127.0.0.1:8262 is alive wait process dm-worker.test exit... got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" check data check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=1 expected=1 dmctl test cmd: "binlog list test" wait process dm-worker.test exit... dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:14435" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:33 CST 2024] <<<<<< finish DM-082 pessimistic >>>>>> got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog inject test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:14627 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique (c);" wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog list test" wait process dm-master.test exit... dmctl test cmd: "binlog skip test" run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:51:34 CST 2024] <<<<<< start DM-085 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" dmctl test cmd: "stop-task test" [Sun May 5 20:51:36 CST 2024] <<<<<< finish DM-LIST_ERROR >>>>>> [Sun May 5 20:51:36 CST 2024] <<<<<< start DM-SKIP_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... [Sun May 5 20:51:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Sun May 5 20:51:36 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Sun May 5 20:51:36 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" use sync_diff_inspector to check increment data check diff successfully check diff successfully [Sun May 5 20:51:37 CST 2024] <<<<<< finish test_multi_task_running >>>>>> [Sun May 5 20:51:37 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join4.toml >>>>>> wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " wait for rpc addr 127.0.0.1:8561 alive the 1-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 123 100 123 0 0 202k 0 --:--:-- --:--:-- --:--:-- 120k wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:37 CST 2024] <<<<<< test case drop_column_with_index success! >>>>>> start running case: [duplicate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:51:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time {"connections":0,"version":"8.0.11-TiDB-v8.2.0-alpha-79-g600b2ed4bf","git_hash":"600b2ed4bf0aa38224a1c4c4c68831820735515c"}dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task-2.yaml " wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:37 CST 2024] <<<<<< test case new_collation_off success! >>>>>> start running case: [only_dml] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-master.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:38 CST 2024] <<<<<< test case downstream_diff_index success! >>>>>> start running case: [downstream_more_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:38 CST 2024] <<<<<< finish DM-085 pessimistic >>>>>> [Sun May 5 20:51:38 CST 2024] <<<<<< start DM-085 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" [Sun May 5 20:51:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:51:39 CST 2024] <<<<<< finish DM-SKIP_ERROR >>>>>> [Sun May 5 20:51:39 CST 2024] <<<<<< start DM-REPLACE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=1 expected=1 check diff successfully read binlog from relay log failed, and will use remote binlog dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/only_dml/source1.yaml" [Sun May 5 20:51:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column new_col text, add column c int; alter table handle_error.tb1 add unique(c);" run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Sun May 5 20:51:40 CST 2024] <<<<<< finish DM-085 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/downstream_more_column/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8461 is alive [Sun May 5 20:51:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/only_dml/source2.yaml" wait process dm-worker.test exit... check log contain failed 1-th time, retry later [Sun May 5 20:51:41 CST 2024] <<<<<< start DM-086 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" run tidb sql failed 2-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-only-ca.yaml" run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:51:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: full mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-master.test exit... dmctl test cmd: "stop-task test" [Sun May 5 20:51:42 CST 2024] <<<<<< finish DM-REPLACE_ERROR >>>>>> [Sun May 5 20:51:42 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchAllDoTablesFailed=return(1152) check log contain failed 2-th time, retry later rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:51:43 CST 2024] <<<<<< finish DM-086 pessimistic >>>>>> [Sun May 5 20:51:43 CST 2024] <<<<<< start DM-086 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" [Sun May 5 20:51:43 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time run tidb sql failed 3-th time, retry later [Sun May 5 20:51:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit check diff successfully PID of insert_data is 5947 dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 dmctl test cmd: "stop-task test" purge relay log failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/downstream_more_column/dm-task-incremental.yaml --remove-meta" rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000002 -s mysql-replica-01" { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-2.yaml --remove-meta=true" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 3-th time, retry later earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 2-th time, retry later wait process dm-worker.test exit... [Sun May 5 20:51:45 CST 2024] <<<<<< finish DM-086 optimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d downstream_more_column1 -t t1 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/data/schema.sql" run tidb sql failed 4-th time, retry later dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=2 expected=2 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000003 -s mysql-replica-01" wait process dm-worker.test exit... [Sun May 5 20:51:46 CST 2024] <<<<<< start DM-094 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 3-th time, retry later { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" reset go failpoints, and need restart dm-worker, then start task again dmctl test cmd: "query-status test2" wait process dm-master.test exit... check data check diff successfully check diff successfully ============================== test_worker_handle_multi_tls_tasks success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Sun May 5 20:51:47 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING pessimistic >>>>>> [Sun May 5 20:51:47 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000004 -s mysql-replica-01" run tidb sql failed 5-th time, retry later wait process dm-worker.test exit... earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 4-th time, retry later check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000005 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=4 expected=4 dmctl test cmd: "stop-task test" [Sun May 5 20:51:49 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING optimistic >>>>>> [Sun May 5 20:51:49 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... run tidb sql failed 6-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" check log contain failed 5-th time, retry later try to isolate leader dm-master3 kill dm-master3 [Sun May 5 20:51:49 CST 2024] <<<<<< finish DM-094 pessimistic >>>>>> [Sun May 5 20:51:49 CST 2024] <<<<<< start DM-094 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000006 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 6-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Sun May 5 20:51:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:51:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000007 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 7-th time, retry later got=4 expected=4 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:50 CST 2024] <<<<<< test case downstream_more_column success! >>>>>> [Pipeline] } wait process dm-master3 exit... Cache not saved (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm already exists) wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Pipeline] } dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml" [Pipeline] // timeout check log contain failed 6-th time, retry later [Pipeline] } rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv run tidb sql failed 7-th time, retry later [Pipeline] } dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000008 -s mysql-replica-01" [Pipeline] // node [Pipeline] } wait process dm-master3 exit... process dm-master3 already exit [Sun May 5 20:51:51 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> try to isolate follower dm-master1 kill dm-master1 [Pipeline] // podTemplate wait process dm-master.test exit... earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 8-th time, retry later [Pipeline] } wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-master.test exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" dmctl test cmd: "validation status test" wait process dm-master1 exit... dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000009 -s mysql-replica-01" [Sun May 5 20:51:53 CST 2024] <<<<<< finish DM-094 optimistic >>>>>> [Sun May 5 20:51:53 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> [Sun May 5 20:51:53 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 9-th time, retry later check log contain failed 7-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 wait process dm-master1 exit... process dm-master1 already exit dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000010 -s mysql-replica-02" [Sun May 5 20:51:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) try to isolate dm-worker1 kill dm-worker1 wait process dm-master.test exit... [Sun May 5 20:51:54 CST 2024] <<<<<< start DM-095 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later wait process dm-worker1 exit... wait process dm-master.test exit... check log contain failed 8-th time, retry later dmctl test cmd: "query-status test" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000011 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 2-th time, retry later wait process dm-worker.test exit... got=2 expected=2 check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: fast mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" got=3 expected=3 check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:56 CST 2024] <<<<<< test case relay_interrupt success! >>>>>> start running case: [safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh... Verbose mode = false wait process dm-worker1 exit... process dm-worker1 already exit [Sun May 5 20:51:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> try to isolate dm-worker2 kill dm-worker2 wait process dm-master.test exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:51:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000013 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "stop-task test" wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ [Sun May 5 20:51:57 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> [Sun May 5 20:51:57 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP pessimistic >>>>>> wait process dm-master.test exit... process dm-master.test already exit add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker2 exit... wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:51:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000014 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 4-th time, retry later check log contain failed 9-th time, retry later wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:51:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> dmctl test cmd: "pause-task test" got=3 expected=3 dmctl test cmd: "resume-task test" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:51:58 CST 2024] <<<<<< finish DM-095 pessimistic >>>>>> [Sun May 5 20:51:58 CST 2024] <<<<<< start DM-095 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" got=3 expected=3 rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:51:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000015 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 5-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " check log contain failed 10-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000016 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... [Sun May 5 20:52:00 CST 2024] <<<<<< finish DM-095 optimistic >>>>>> dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000017 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 7-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Sun May 5 20:52:01 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP pessimistic >>>>>> [Sun May 5 20:52:01 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task.yaml --remove-meta" check log contain failed 11-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:52:01 CST 2024] <<<<<< start DM-096 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000018 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 8-th time, retry later check diff successfully wait process dm-worker.test exit... [Sun May 5 20:52:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Sun May 5 20:52:02 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Sun May 5 20:52:02 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:52:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-worker.test exit... check log contain failed 12-th time, retry later dmctl test cmd: "pause-task test" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000019 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 9-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "resume-task test" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000020 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 1-th time, retry later wait process dm-worker.test exit... got=3 expected=3 check diff failed 1-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again check log contain failed 13-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Sun May 5 20:52:05 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP optimistic >>>>>> [Sun May 5 20:52:05 CST 2024] <<<<<< start DM-SKIP_INCOMPATIBLE_DDL >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:05 CST 2024] <<<<<< start test_COMMIT_in_QueryEvent >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000021 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 2-th time, retry later [Sun May 5 20:52:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/start_task/1/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Sun May 5 20:52:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000022 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 3-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/start_task/1/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh: line 556: hello: command not found dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "binlog skip test" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:52:07 CST 2024] <<<<<< finish DM-096 pessimistic >>>>>> [Sun May 5 20:52:07 CST 2024] <<<<<< start DM-096 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" check log contain failed 14-th time, retry later check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-worker.test exit... [Sun May 5 20:52:07 CST 2024] <<<<<< finish DM-SKIP_INCOMPATIBLE_DDL >>>>>> [Sun May 5 20:52:07 CST 2024] <<<<<< start DM-4202 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check update pk(split into insert and delete) dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000024 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 4-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/start_task/1/dm-task.yaml --remove-meta=true" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "pause-task test" wait process dm-worker.test exit... dmctl test cmd: "resume-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 1 [Sun May 5 20:52:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000024 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" [Sun May 5 20:52:07 CST 2024] <<<<<< finish DM-RemoveLock optimistic >>>>>> wait process dm-master exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 7\/1\/2" count: 0 != expected: 1, failed the 0-th time, will retry again start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:35724" dmctl test cmd: "query-status test" wait process dm-master exit... process dm-master already exit [Sun May 5 20:52:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 15-th time, retry later [Sun May 5 20:52:09 CST 2024] <<<<<< finish DM-096 optimistic >>>>>> wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff failed 1-th time, retry later [Sun May 5 20:52:09 CST 2024] <<<<<< finish DM-4202 >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:52:10 CST 2024] <<<<<< start DM-097 pessimistic >>>>>> dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000026 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" [Sun May 5 20:52:10 CST 2024] <<<<<< start DM-4204 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... check data check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:52:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time start dm-worker2 and kill dm-worker1 dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000027 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 7-th time, retry later got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:11 CST 2024] <<<<<< test case many_tables success! >>>>>> start running case: [online_ddl] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 16-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" got=2 expected=2 [Sun May 5 20:52:12 CST 2024] <<<<<< start DM-RestartMaster pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker1 exit... check diff successfully <<<<<< test_COMMIT_in_QueryEvent success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000028 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 8-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... [Sun May 5 20:52:12 CST 2024] <<<<<< finish DM-4204 >>>>>> wait process dm-worker1 exit... process dm-worker1 already exit wait process dm-master exit... mysql_ssl_setup at=/var/lib/mysql/ [Sun May 5 20:52:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/tls/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator panic and we can catch it 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000029 -s mysql-replica-01" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 9-th time, retry later check log contain failed 17-th time, retry later wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:52:13 CST 2024] <<<<<< start DM-4206 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source1.yaml" dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" wait process dm-master exit... process dm-master already exit [Sun May 5 20:52:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" wait process dm-master.test exit... dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000030 -s mysql-replica-02" [Sun May 5 20:52:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" resume task to worker2 dmctl test cmd: "resume-task test" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 1-th time, retry later check diff successfully wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source2.yaml" check diff successfully ============================== test_worker_ha_when_enable_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive check log contain failed 18-th time, retry later dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:39469 -s mysql-replica-01" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000031 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 2-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:39276 -s mysql-replica-01" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:36408 -s mysql-replica-02" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:36627 -s mysql-replica-02" dmctl test cmd: "query-status test" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:52:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/online_ddl/dm-task.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:16 CST 2024] <<<<<< test case start_task success! >>>>>> start running case: [print_status] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh... Verbose mode = false +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status ++ pwd + cur=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/../_utils/test_prepare ++ source env_variables +++ MYSQL_HOST1=127.0.0.1 +++ MYSQL_HOST2=127.0.0.1 +++ TIDB_HOST=127.0.0.1 +++ MYSQL_PORT1=3306 +++ MYSQL_PORT2=3307 +++ MYSQL_PASSWORD1=123456 +++ MYSQL_PASSWORD2=123456 +++ TIDB_PASSWORD=123456 +++ TIDB_PORT=4000 +++ MASTER_PORT=8261 +++ MASTER_PEER_PORT=8291 +++ MASTER_PORT1=8261 +++ MASTER_PORT2=8361 +++ MASTER_PORT3=8461 +++ MASTER_PORT4=8561 +++ MASTER_PORT5=8661 +++ MASTER_PORT6=8761 +++ MASTER_PEER_PORT1=8291 +++ MASTER_PEER_PORT2=8292 +++ MASTER_PEER_PORT3=8293 +++ MASTER_PEER_PORT4=8294 +++ MASTER_PEER_PORT5=8295 +++ MASTER_PEER_PORT6=8296 +++ WORKER1_PORT=8262 +++ WORKER2_PORT=8263 +++ WORKER3_PORT=8264 +++ WORKER4_PORT=18262 +++ WORKER5_PORT=18263 +++ WORKER1_NAME=worker1 +++ WORKER2_NAME=worker2 +++ SOURCE_ID1=mysql-replica-01 +++ SOURCE_ID2=mysql-replica-02 +++ RESET_MASTER=true +++ VERBOSE=false ++ '[' true = true ']' ++ run_sql 'RESET MASTER' 3306 123456 ++ run_sql 'RESET MASTER' 3307 123456 + WORK_DIR=/tmp/dm_test/print_status + cleanup_data print_status + rm -rf /tmp/dm_test/print_status + mkdir /tmp/dm_test/print_status + for target_db in '"$@"' + run_sql 'drop database if exists `print_status`' 4000 123456 + run_sql 'drop database if exists dm_meta' 4000 123456 + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=0 + echo '0 dm-master alive' 0 dm-master alive + pkill -hup dm-master.test + true ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=0 + echo '0 dm-worker alive' 0 dm-worker alive + pkill -hup dm-worker.test + true ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test process dm-master.test already exit + wait_process_exit dm-worker.test process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + run + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.prepare.sql 127.0.0.1 3306 123456 + inject_points=("github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)" "github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)" "github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)") ++ join_string ';' 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)' 'github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)' 'github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' ++ local 'IFS=;' ++ shift ++ echo 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + GO_FAILPOINTS='github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + cp /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-worker1.toml /tmp/dm_test/print_status/dm-worker1.toml + sed -i s%placeholder%/tmp/dm_test/print_status/relay_by_worker%g /tmp/dm_test/print_status/dm-worker1.toml + run_dm_master /tmp/dm_test/print_status/master 8261 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml [Sun May 5 20:52:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_master_online 127.0.0.1:8261 [Sun May 5 20:52:16 CST 2024] <<<<<< finish DM-4206 pessimistic >>>>>> [Sun May 5 20:52:16 CST 2024] <<<<<< start DM-4206 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master exit... dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000032 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 3-th time, retry later [Sun May 5 20:52:17 CST 2024] <<<<<< finish DM-097 pessimistic >>>>>> [Sun May 5 20:52:17 CST 2024] <<<<<< start DM-097 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 19-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000033 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 4-th time, retry later rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:52:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time use sync_diff_inspector to check full dump data check diff successfully alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 wait process dm-master.test exit... check diff successfully dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:41438 -s mysql-replica-01" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive + run_dm_worker /tmp/dm_test/print_status/worker1 8262 /tmp/dm_test/print_status/dm-worker1.toml got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:41245 -s mysql-replica-01" dmctl test cmd: "query-status test" [Sun May 5 20:52:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/print_status/dm-worker1.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_worker_online 127.0.0.1:8262 wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:38566 -s mysql-replica-02" dmctl test cmd: "query-status test" check log contain failed 20-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000034 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 5-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:38785 -s mysql-replica-02" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master exit... got=3 expected=3 dmctl test cmd: "stop-task test" [Sun May 5 20:52:20 CST 2024] <<<<<< finish DM-4206 optimistic >>>>>> alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 rpc addr 127.0.0.1:8262 is alive + cp /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/source1.yaml /tmp/dm_test/print_status/source1.yaml + sed -i '/relay-binlog-name/i\relay-dir: /tmp/dm_test/print_status/worker1/relay_log' /tmp/dm_test/print_status/source1.yaml + dmctl_operate_source create /tmp/dm_test/print_status/source1.yaml mysql-replica-01 + operate=create + source_toml=/tmp/dm_test/print_status/source1.yaml + source_id=mysql-replica-01 + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'operate-source create /tmp/dm_test/print_status/source1.yaml' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "operate-source create /tmp/dm_test/print_status/source1.yaml" dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000035 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 6-th time, retry later wait process dm-worker.test exit... [Sun May 5 20:52:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time + dmctl_start_task_standalone + '[' 0 -ge 2 ']' + remove_meta= + '[' 0 -ge 1 ']' + task_conf=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml ' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml " wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:52:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master exit... process dm-master already exit [Sun May 5 20:52:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:52:21 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit success success [Sun May 5 20:52:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:52:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000037 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 7-th time, retry later + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check diff successfully + ls /tmp/dm_test/print_status/relay_by_worker/worker1/9a6a9351-0add-11ef-be0e-2a546cdaac3c.000001 /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/9a6a9351-0add-11ef-be0e-2a546cdaac3c.000001: dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25-bin.000001 relay.meta + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.increment.sql 127.0.0.1 3306 123456 + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check log contain failed 21-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000038 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 8-th time, retry later check diff failed 1-th time, retry later alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:43214" dmctl test cmd: "query-status test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:40724" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "query-status test" got=0 expected=1 command: validation status test validator panic count: 0 != expected: 1, failed the 0-th time, will retry again got=3 expected=3 dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "stop-task test" [Sun May 5 20:52:23 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Sun May 5 20:52:23 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... check log contain failed 22-th time, retry later [Sun May 5 20:52:23 CST 2024] <<<<<< finish DM-097 optimistic >>>>>> dmctl test cmd: "purge-relay --filename dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000039 -s mysql-replica-02" earliest_relay_log1: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 earliest_relay_log2: dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001 purge relay log failed 9-th time, retry later rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:52:24 CST 2024] <<<<<< finish DM-RestartMaster pessimistic >>>>>> [Sun May 5 20:52:24 CST 2024] <<<<<< start DM-RestartMaster optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" check diff successfully + check_log_contains /tmp/dm_test/print_status/worker1/log/dm-worker.log 'enable safe-mode because of task initialization.*duration=1m0s' + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=1 + echo '1 dm-master alive' 1 dm-master alive + pkill -hup dm-master.test ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=1 + echo '1 dm-worker alive' 1 dm-worker alive + pkill -hup dm-worker.test ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:24 CST 2024] <<<<<< start DM-098 pessimistic >>>>>> check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh: line 44: 5947 Terminated insert_data 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:44990" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:42663" wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check log contain failed 23-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" got=1 expected=1 --> check validator worker panic and we can catch it 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully check log contain failed 1-th time, retry later [Sun May 5 20:52:26 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> [Sun May 5 20:52:26 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-master.test exit... process dm-master.test already exit + wait_process_exit dm-worker.test wait process dm-master.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check log contain failed 24-th time, retry later wait process dm-worker1 exit... got=2 expected=2 wait process dm-worker.test exit... dmctl test cmd: "query-status test" dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:47024" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:44602" ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. wait process dm-worker.test exit... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' [Sun May 5 20:52:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master-no-tls.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=2 expected=2 got=1 expected=1 restart dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-worker1 exit... process dm-worker1 already exit [Sun May 5 20:52:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker.test exit... [Sun May 5 20:52:28 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Sun May 5 20:52:28 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time success success finish running run safe mode recover case 1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. check log contain failed 25-th time, retry later wait process dm-worker.test exit... wait process dm-master exit... rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + check_print_status + i=0 + '[' 0 -lt 3 ']' ++ grep 'dm-worker exit' /tmp/dm_test/print_status/worker1/log/dm-worker.log + exit_log='[2024/05/05 20:52:27.397 +08:00] [INFO] [main.go:94] ["dm-worker exit"]' + '[' '[2024/05/05 20:52:27.397 +08:00] [INFO] [main.go:94] ["dm-worker exit"]' == 'not found' ']' + break + '[' 0 -ge 3 ']' + echo 'checking print status' checking print status + dump_status_file=/tmp/dm_test/print_status/worker1/log/dump_status.log + grep -o 'progress status of dumpling' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=1 + '[' 1 -ge 1 ']' + grep -o '\[bps=0' /tmp/dm_test/print_status/worker1/log/dm-worker.log + true ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=0 + '[' 0 -eq 0 ']' + status_file=/tmp/dm_test/print_status/worker1/log/loader_status.log + grep -oP '\[unit=lightning-load\] \[IsCanceled=false\] \[finished_bytes=59674\] \[total_bytes=59674\] \[progress=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/loader_status.log ++ awk '{print $1}' + status_count=1 + '[' 1 -eq 1 ']' + grep 'current speed (bytes / seconds)' /tmp/dm_test/print_status/worker1/log/dm-worker.log + grep -vq '"current speed (bytes / seconds)"=0' + echo 'check load unit print status success' check load unit print status success + status_file2=/tmp/dm_test/print_status/worker1/log/syncer_status.log + grep -oP '\[total_rows=[0-9]+\] \[total_rps=[0-9]+\] \[rps=[0-9]+\] \[master_position=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/syncer_status.log ++ awk '{print $1}' + status_count2=9 + '[' 9 -ge 1 ']' + echo 'check sync unit print status success' check sync unit print status success ++ date + echo '[Sun May 5 20:52:29 CST 2024] <<<<<< test case print_status success! >>>>>>' [Sun May 5 20:52:29 CST 2024] <<<<<< test case print_status success! >>>>>> start running case: [http_apis] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:29 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /tmp/dm_test/tls/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 [Sun May 5 20:52:29 CST 2024] <<<<<< finish DM-098 pessimistic >>>>>> wait process dm-master.test exit... [Sun May 5 20:52:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:49058" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:30 CST 2024] <<<<<< test case only_dml success! >>>>>> start running case: [openapi] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master exit... process dm-master already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:46541" dmctl test cmd: "query-status test" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 got=3 expected=3 dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sun May 5 20:52:31 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-no-tls.yaml" check log contain failed 26-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... [Sun May 5 20:52:32 CST 2024] <<<<<< start DM-4209 pessimistic >>>>>> got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-no-tls.yaml --remove-meta=true" [Sun May 5 20:52:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:32 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:52:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" rpc addr 127.0.0.1:8262 is alive source-id: mysql-replica-01\nflavor: ''\nenable-gtid: true\nenable-relay: true\nrelay-dir: /tmp/dm_test/http_apis/worker1/relay_log\nrelay-binlog-name: ''\nrelay-binlog-gtid: ''\nfrom:\n host: 127.0.0.1\n user: root\n password: '123456'\n port: 3306\n % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 446 100 166 100 280 1277 2154 --:--:-- --:--:-- --:--:-- 2170 start task and check stage ---\nname: test\ntask-mode: all\nis-sharding: false\nmeta-schema: \"dm_meta\"\n# enable-heartbeat: true\n\ntarget-database:\n host: \"127.0.0.1\"\n port: 4000\n user: \"root\"\n password: \"\"\n\nmysql-instances:\n - source-id: \"mysql-replica-01\"\n block-allow-list: \"instance\"\n mydumper-config-name: \"global\"\n loader-config-name: \"global\"\n syncer-config-name: \"global\"\n\nblock-allow-list:\n instance:\n do-dbs: [\"http_apis\"]\n\nmydumpers:\n global:\n threads: 4\n chunk-filesize: 0\n skip-tz-utc: true\n statement-size: 100\n extra-args: \"\"\n\nloaders:\n global:\n pool-size: 16\n dir: \"./dumped_data\"\n\nsyncers:\n global:\n worker-count: 16\n batch: 100\n HTTP 127.0.0.1:8261/apis/v1alpha1/status/test-task is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed got=2 expected=2 got=1 expected=1 check data check diff successfully ============================== test_source_and_target_with_empty_tlsconfig success ================================== check log contain failed 27-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 2 [Sun May 5 20:52:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 742 0 0 100 742 0 740 0:00:01 0:00:01 --:--:-- 741 100 950 100 208 100 742 191 681 0:00:01 0:00:01 --:--:-- 681 wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Sun May 5 20:52:22 CST 2024] <<<<<< finish test_kill_master_in_sync >>>>>> [Sun May 5 20:52:22 CST 2024] <<<<<< start test_kill_worker_in_sync >>>>>> [Sun May 5 20:52:22 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... [Sun May 5 20:52:35 CST 2024] <<<<<< finish DM-RestartMaster optimistic >>>>>> [Sun May 5 20:52:35 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" pt-online-schema-change 3.0.12 wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:52:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 28-th time, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 287k 0 --:--:-- --:--:-- --:--:-- 301k get sub task configs % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2299 0 2299 0 0 583k 0 --:--:-- --:--:-- --:--:-- 748k pause task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 194 100 183 100 11 16187 973 --:--:-- --:--:-- --:--:-- 16636 dmctl test cmd: "stop-task test" [Sun May 5 20:52:36 CST 2024] <<<<<< finish DM-4209 pessimistic >>>>>> [Sun May 5 20:52:36 CST 2024] <<<<<< start DM-4209 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:36 CST 2024] <<<<<< test case shardddl3 success! >>>>>> start running case: [shardddl3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:52:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1960 100 1960 0 0 309k 0 --:--:-- --:--:-- --:--:-- 319k resume task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 195 100 184 100 11 9393 561 --:--:-- --:--:-- --:--:-- 9684 rpc addr 127.0.0.1:8263 is alive use sync_diff_inspector to check full data check diff successfully alter: add column age int; 1.1.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" check log contain failed 29-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 298k 0 --:--:-- --:--:-- --:--:-- 362k got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" run tidb sql failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source1.yaml" [Sun May 5 20:52:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 801 100 801 0 0 122k 0 --:--:-- --:--:-- --:--:-- 130k alter: add column age int; 1.1.0 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit check diff successfully check diff failed 1-th time, retry later check log contain failed 30-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source2.yaml" kill dm-worker1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" wait process dm-master.test exit... dmctl test cmd: "stop-task test" [Sun May 5 20:52:40 CST 2024] <<<<<< finish DM-4209 optimistic >>>>>> got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... [Sun May 5 20:52:40 CST 2024] <<<<<< start DM-099 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker1 exit... check diff successfully wait process dm-worker.test exit... alter: add key name (name); 1.1.0 check log contain failed 1-th time, retry later wait process dm-master.test exit... check diff failed 1-th time, retry later [Sun May 5 20:52:41 CST 2024] <<<<<< start DM-4211 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:41 CST 2024] <<<<<< test case tls success! >>>>>> wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --worker --name=worker1" got=1 expected=1 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 7840 0 --:--:-- --:--:-- --:--:-- 8250 check data wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:52:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8361 alive the 2-th time check diff successfully [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 [Pipeline] // withCredentials [Pipeline] } rpc addr 127.0.0.1:8361 is alive [Sun May 5 20:52:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator stop when pending row size too large 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog replace test --binlog-pos dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:54386 alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" wait process dm-master.test exit... alter: add key name (name); 1.1.0 [Pipeline] // container wait process dm-master.test exit... check diff successfully [Pipeline] } [Pipeline] // withEnv [Pipeline] } got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker2 exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:52:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] // node check diff failed 1-th time, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Sun May 5 20:52:43 CST 2024] <<<<<< finish DM-4211 >>>>>> wait process dm-master.test exit... [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff successfully [Sun May 5 20:52:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:44 CST 2024] <<<<<< test case http_apis success! >>>>>> start running case: [new_relay] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:52:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "pause-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" wait process dm-master.test exit... [Sun May 5 20:52:44 CST 2024] <<<<<< start DM-4213 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:52:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: RELAY ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) dmctl test cmd: "query-status -s mysql-01" wait process dm-master.test exit... got=2 expected=2 got=1 expected=1 ('enable_relay_failed resp=', {u'error_code': 46004, u'error_msg': u'[code=46004:class=scheduler:scope=internal:level=medium], Message: dm-worker with name [no-worker] not exists'}) alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 rpc addr 127.0.0.1:8263 is alive check diff successfully check diff successfully wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" [Sun May 5 20:52:46 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS pessimistic >>>>>> ('get_source_status_failed resp=', {u'error_code': 46008, u'error_msg': u'[code=46008:class=scheduler:scope=internal:level=medium], Message: source config with ID no-mysql not exists'}) [Sun May 5 20:52:46 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... [Sun May 5 20:52:46 CST 2024] <<<<<< finish DM-099 pessimistic >>>>>> ('get_source_status_success resp=', {u'total': 1, u'data': [{u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 1, u'data': [{u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('disable_relay_failed resp=', {u'error_code': 46004, u'error_msg': u'[code=46004:class=scheduler:scope=internal:level=medium], Message: dm-worker with name [no-worker] not exists'}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 ('get_source_status_success_no_relay resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) [Sun May 5 20:52:46 CST 2024] <<<<<< finish DM-4213 >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" wait process dm-worker.test exit... ('list_source_with_status_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'task_name_list': [], u'flavor': u'mysql', u'password': u'******', u'port': 3306, u'status_list': [{u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'', u'relay_catch_up_master': False, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}]}) ('get_source_status_success resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" ('get_source_status_success_with_relay resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'96ae8793-0add-11ef-bf03-aa1057f7af0c.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-2', u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... process dm-worker.test already exit success success [Sun May 5 20:52:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:52:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:52:47 CST 2024] <<<<<< start DM-100 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 delete_source_success >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: RELAY SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: SOURCE ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source_failed resp=', {u'error_code': 10001, u'error_msg': u'[code=10001:class=database:scope=upstream:level=high], Message: database driver error, RawCause: dial tcp :0: connect: connection refused, Workaround: Please check the database connection and the database config in configuration file.'}) ('update_source1_without_password_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': None, u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('list_source_with_status_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'task_name_list': [], u'flavor': u'mysql', u'password': u'******', u'port': 3306, u'status_list': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}]}) got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-master.test exit... ('get_source_schemas_and_tables_success schema_resp=', [u'information_schema', u'ha_test', u'import_v10x', u'mysql', u'only_dml', u'openapi', u'performance_schema', u'sharding22', u'sys']) ('get_source_schemas_and_tables_success table_resp=', [u't1']) ('get_source_schemas_and_tables_success schema_resp=', [u'information_schema', u'db-name', u'ha_test', u'import_v10x', u'mysql', u'only_dml', u'openapi', u'performance_schema', u'sharding22', u'sys']) ('get_source_schemas_and_tables_success table_resp=', [u't1']) dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully delete_source_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time ('delete_source_failed msg=', {u'error_code': 46008, u'error_msg': u'[code=46008:class=scheduler:scope=internal:level=medium], Message: source config with ID mysql-01 not exists'}) ('list_source_with_reverse resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: SOURCE SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: SHARD TASK ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 check log contain failed 1-th time, retry later ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) wait process dm-master.test exit... ('create_task_failed resp=', {u'error': u'error in openapi3filter.RequestError: request body has an error: doesn\'t match the schema: Error at "/task/shard_mode": value is not one of the allowed values'}) wait process dm-master.test exit... process dm-master.test already exit ('create_shard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'binlog_filter_rule': {u'rule-2': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}, u'rule-1': {u'ignore_event': [u'delete']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'rule-1'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'rule-2'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-shard" got=2 expected=2 check diff successfully check diff failed 1-th time, retry later wait process dm-worker1 exit... alter: add column age int; 1.1.0 dmctl test cmd: "query-status test-shard" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully ('get_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name not a task name not exist, Workaround: Please use `query-status` command to see tasks.'}) wait process dm-master.test exit... ('get_illegal_char_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name t-\xcb!s`t not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 6, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3103)', u'syncer_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-12', u'synced': False, u'total_tps': 0, u'unresolved_groups': [{u'synced': None, u'first_location': u'', u'unsynced': [u"this DM-worker doesn't receive any shard DDL of this group"], u'target': u'`openapi`.`t`', u'ddl_list': None}], u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 2344)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': [u'ALTER TABLE `openapi`.`t` ADD COLUMN `aaa` INT'], u'recent_tps': 0, u'total_events': 6, u'master_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-11', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 2511)', u'syncer_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-10', u'synced': False, u'total_tps': 0, u'unresolved_groups': [{u'synced': [u'`openapi`.`t`'], u'first_location': u'position: (dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 2384), gtid-set: 9707fb65-0add-11ef-aee9-aa1057f7af0c:1-10', u'unsynced': None, u'target': u'`openapi`.`t`', u'ddl_list': [u'ALTER TABLE `openapi`.`t` ADD COLUMN `aaa` INT']}], u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 2307)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'binlog_filter_rule': {u'mysql-01-filter-rule-0': {u'ignore_event': [u'delete']}, u'mysql-02-filter-rule-0': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-01-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-02-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}]}) check diff failed 2-th time, retry later alter: add column age int; 1.1.0 wait process dm-worker.test exit... wait process dm-worker1 exit... process dm-worker1 already exit [Sun May 5 20:52:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'binlog_filter_rule': {u'mysql-01-filter-rule-0': {u'ignore_event': [u'delete']}, u'mysql-02-filter-rule-0': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-01-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-02-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}]}) rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) check log contain failed 2-th time, retry later [Sun May 5 20:52:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time delete_source_with_force_success delete_source_with_force_success wait process dm-master.test exit... wait process dm-worker.test exit... ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: SHARD TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: MULTI TASK dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" check diff successfully ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) check diff successfully ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}}) dmctl test cmd: "query-status test-1" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:52:52 CST 2024] <<<<<< finish DM-100 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "query-status test-1" [Sun May 5 20:52:52 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS optimistic >>>>>> [Sun May 5 20:52:52 CST 2024] <<<<<< start DM-COLUMN_INDEX pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" check diff failed 3-th time, retry later got=2 expected=2 ('get_task_list_with_status resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3891)', u'syncer_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3891)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3395)', u'syncer_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3395)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-1'}]}) ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-2'}}) dmctl test cmd: "query-status test-2" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:52 CST 2024] <<<<<< test case handle_error success! >>>>>> start running case: [handle_error_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:52:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test-2" got=2 expected=2 alter: add key name (name); 1.1.0 [Sun May 5 20:52:53 CST 2024] <<<<<< start DM-101 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... ('get_task_list_with_status resp=', {u'total': 2, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3891)', u'syncer_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3891)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3395)', u'syncer_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3395)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-1'}, {u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-02'}, {u'source_name': u'mysql-01'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-2', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3891)', u'syncer_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3891)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-2', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3395)', u'syncer_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 3395)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-2'}]}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: MULTI TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: NO SHARD TASK check log contain failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) wait process dm-master.test exit... check diff failed 4-th time, retry later check diff failed 1-th time, retry later ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}}) dmctl test cmd: "query-status test-no-shard" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:52:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker1.toml >>>>>> dmctl test cmd: "query-status test-no-shard" got=2 expected=2 alter: add key name (name); 1.1.0 wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully ('get_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name not a task name not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-02', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 4279)', u'syncer_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-18', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 4005)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-01', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-23', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 4679)', u'syncer_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-22', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 4425)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}]}) delete_source_with_force_success ('get_task_status_success resp=', {u'total': 1, u'data': [{u'source_name': u'mysql-02', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 4279)', u'syncer_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-18', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 4005)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}]}) dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... check log contain failed 4-th time, retry later ('get_task_schema_success schema resp=', [u'openapi']) ('get_task_schema_success table resp=', [u't2']) ('get_task_schema_success create table resp=', {u'table_name': u't2', u'schema_create_sql': u'CREATE TABLE `t2` ( `i` tinyint(4) DEFAULT NULL, `j` int(11) DEFAULT NULL, UNIQUE KEY `j` (`j`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin', u'schema_name': u'openapi'}) ('get_task_schema_success table resp=', [u't2']) wait process dm-worker2 exit... ('delete_task_failed resp=', {u'error_code': 49001, u'error_msg': u'[code=49001:class=openapi:scope=internal:level=high], Message: task test-no-shard have running subtasks, please stop them or delete task with force.'}) rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source1.yaml" delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) [Sun May 5 20:52:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time delete_source_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: NO SHARD TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: TASK TEMPLATES wait process dm-master.test exit... ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:52:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:52:56 CST 2024] <<<<<< finish DM-COLUMN_INDEX pessimistic >>>>>> [Sun May 5 20:52:56 CST 2024] <<<<<< start DM-COLUMN_INDEX optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success success finish running run safe mode recover case 2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed 5-th time, retry later ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('create_task_template_failed resp=', {u'error': u'error in openapi3filter.RequestError: request body has an error: doesn\'t match the schema: Error at "/shard_mode": value is not one of the allowed values'}) ('create_task_template_success resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}) rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source2.yaml" ('list_task_template resp=', {u'total': 1, u'data': [{u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}]}) ('get_task_template resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}) got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again wait process dm-master.test exit... >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_leak passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive ('update_task_template_success resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'full', u'name': u'test-1'}) delete_task_template ('list_task_template resp=', {u'total': 0, u'data': []}) wait process dm-master.test exit... alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}}) dmctl test cmd: "query-status test-1" rpc addr 127.0.0.1:8263 is alive got=2 expected=2 dmctl test cmd: "query-status test-1" check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 ('import_task_template resp=', {u'success_task_list': [u'test-1'], u'failed_task_list': []}) ('list_task_template resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-02'}, {u'source_name': u'mysql-01'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}]}) dmctl test cmd: "config task test-1 --path /tmp/dm_test/openapi/get_task_from_task.yaml" dmctl test cmd: "query-status test" got=2 expected=2 [Sun May 5 20:52:58 CST 2024] <<<<<< finish DM-101 optimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "config task-template test-1 --path /tmp/dm_test/openapi/get_task_from_task_template.yaml" [Sun May 5 20:52:58 CST 2024] <<<<<< start DM-4215 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" delete_source_with_force_success delete_source_with_force_success check diff failed 6-th time, retry later check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: TASK TEMPLATES >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: NO SHARD TASK DUMP STATUS [Sun May 5 20:52:59 CST 2024] <<<<<< finish DM-COLUMN_INDEX optimistic >>>>>> [Sun May 5 20:52:59 CST 2024] <<<<<< start DM-102 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 wait process dm-worker.test exit... wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1+ got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator stop when pending row count too many 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker exit... got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:53:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker exit... process dm-worker already exit [Sun May 5 20:53:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sun May 5 20:53:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 3 [Sun May 5 20:53:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb`" alter: add column age int; pt-online-schema-change 3.0.12 alter: add column age int; pt-online-schema-change 3.0.12 rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:53:02 CST 2024] <<<<<< start DM-DML_EXECUTE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit alter: add key name (name); pt-online-schema-change 3.0.12 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... alter: add key name (name); got=1 expected=1 rpc addr 127.0.0.1:8263 is alive pt-online-schema-change 3.0.12 dmctl test cmd: "stop-task test" ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; [Sun May 5 20:53:03 CST 2024] <<<<<< finish DM-102 >>>>>> wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) pt-online-schema-change 3.0.12 dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait process dm-worker.test exit... ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard-dump-status'}}) alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; pt-online-schema-change 3.0.12 wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test-no-shard-dump-status" [Sun May 5 20:53:04 CST 2024] <<<<<< start DM-103 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... alter: add column age int; pt-online-schema-change 3.0.12 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time alter: add column age int; got=2 expected=2 dmctl test cmd: "query-status test-no-shard-dump-status" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time pt-online-schema-change 3.0.12 got=2 expected=2 got=2 expected=2 ('check_dump_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-02', u'name': u'test-no-shard-dump-status', u'unresolved_ddl_lock_id': u'', u'worker_name': u'worker2', u'dump_status': {u'estimate_total_rows': 0, u'finished_bytes': 0, u'completed_tables': 0, u'total_tables': 0, u'finished_rows': 0}, u'unit': u'Dump', u'stage': u'Running'}, {u'source_name': u'mysql-01', u'name': u'test-no-shard-dump-status', u'unresolved_ddl_lock_id': u'', u'worker_name': u'worker1', u'dump_status': {u'estimate_total_rows': 0, u'finished_bytes': 0, u'completed_tables': 0, u'total_tables': 0, u'finished_rows': 0}, u'unit': u'Dump', u'stage': u'Running'}]}) alter: add key name (name); wait process dm-master.test exit... pt-online-schema-change 3.0.12 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 alter: add key name (name); pt-online-schema-change 3.0.12 got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 1-th time, retry later wait process dm-worker.test exit... [Sun May 5 20:53:06 CST 2024] <<<<<< finish DM-DML_EXECUTE_ERROR >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; pt-online-schema-change 3.0.12 alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:53:06 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time pt-online-schema-change 3.0.12 wait process dm-worker.test exit... wait process dm-worker exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:53:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time process dm-worker.test already exit ('get_task_status_success_but_worker_meet_error resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-no-shard-dump-status', u'worker_name': u'', u'error_msg': u'[code=38029:class=dm-master:scope=internal:level=high], Message: mysql-01 relevant worker-client not found, Workaround: Please use list-member command to see if the some workers are offline.', u'unit': u'', u'stage': u''}, {u'source_name': u'mysql-02', u'name': u'test-no-shard-dump-status', u'worker_name': u'', u'error_msg': u'[code=38029:class=dm-master:scope=internal:level=high], Message: mysql-02 relevant worker-client not found, Workaround: Please use list-member command to see if the some workers are offline.', u'unit': u'', u'stage': u''}]}) Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-worker.test already exit [Sun May 5 20:53:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time use sync_diff_inspector to check increment data check diff successfully start dm-worker3 and kill dm-worker2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1 worker2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker2 exit... wait process dm-worker exit... process dm-worker already exit [Sun May 5 20:53:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sun May 5 20:53:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:53:08 CST 2024] <<<<<< finish DM-103 pessimistic >>>>>> [Sun May 5 20:53:08 CST 2024] <<<<<< start DM-103 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:53:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:53:09 CST 2024] <<<<<< start DM-KEY_NOT_FOUND >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "transfer-source mysql-replica-01 worker1" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=3 expected=3 got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) delete_source_with_force_success check log contain failed 1-th time, retry later wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:53:09 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: NO SHARD TASK DUMP STATUS SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: COMPLEX OPERATION ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) check diff failed 1-th time, retry later ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-complex'}}) dmctl test cmd: "query-status test-complex" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully rpc addr 127.0.0.1:8264 is alive wait and check task running dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-complex'}]}) dmctl test cmd: "query-status test-complex" check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:53:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 check diff successfully [Sun May 5 20:53:11 CST 2024] <<<<<< finish DM-KEY_NOT_FOUND >>>>>> ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-complex', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-33', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 6589)', u'syncer_binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-32', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 6335)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-complex', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-29', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 6441)', u'syncer_binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-28', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001, 6167)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}]}) check diff successfully check log contain failed 2-th time, retry later check diff failed 1-th time, retry later wait process dm-master.test exit... check diff successfully delete_source_with_force_success delete_source_with_force_success rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker exit... ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: COMPLEX OPERATION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: TEST TASK WITH IGNORE CHECK ITEMS ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) wait process dm-master.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:53:12 CST 2024] <<<<<< finish DM-103 optimistic >>>>>> dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) ('create_task_with_precheck resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'name': u'test-no-ignore-no-error', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u''], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-no-ignore-no-error" got=2 expected=2 use sync_diff_inspector to check increment2 data now! check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive delete_task_success dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" wait process dm-master.test exit... wait process dm-worker exit... process dm-worker already exit [Sun May 5 20:53:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sun May 5 20:53:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:53:13 CST 2024] <<<<<< start DM-104 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_task_with_precheck resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 21,\n\t\t\t\t"name": "sharding table `openapi`.`t` consistency checking",\n\t\t\t\t"desc": "check consistency of sharding table structures",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "sourceID mysql-01 table {openapi t} of sharding `openapi`.`t` have auto-increment key, please make sure them don\'t conflict in target table!"\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "sourceID mysql-02 table {openapi t} of sharding `openapi`.`t` have auto-increment key, please make sure them don\'t conflict in target table!"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "If happen conflict, please handle it by yourself. You can refer to https://docs.pingcap.com/tidb-data-migration/stable/shard-merge-best-practices/#handle-conflicts-between-primary-keys-or-unique-indexes-across-multiple-sharded-tables",\n\t\t\t\t"extra": "auto-increment key checking"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 22,\n\t\t\t"successful": 21,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'name': u'test-no-ignore-has-warn', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u''], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-no-ignore-has-warn" check log contain failed 3-th time, retry later check diff successfully kill dm-worker1 wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:53:14 CST 2024] <<<<<< start DM-001 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" got=2 expected=2 delete_task_success got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again wait process dm-worker1 exit... wait process dm-master.test exit... ('get_task_list resp=', {u'total': 0, u'data': []}) wait process dm-worker.test exit... ('create_task_with_precheck resp=', {u'error_code': 26005, u'error_msg': u'[code=26005:class=dm-master:scope=internal:level=medium], Message: fail to check synchronization configuration with type: check was failed, please see detail\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 21,\n\t\t\t\t"name": "sharding table `openapi`.`t` consistency checking",\n\t\t\t\t"desc": "check consistency of sharding table structures",\n\t\t\t\t"state": "fail",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "column length mismatch (self: 3 vs other: 2)",\n\t\t\t\t\t\t"self": "sourceID mysql-01 table {openapi t} columns [id i j]",\n\t\t\t\t\t\t"other": "sourceID mysql-02 table `openapi`.`t` columns [i j]"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "please set same table structure for sharding tables",\n\t\t\t\t"extra": "error on sharding `openapi`.`t`"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": false,\n\t\t\t"total": 22,\n\t\t\t"successful": 21,\n\t\t\t"failed": 1,\n\t\t\t"warning": 0\n\t\t}\n\t}'}) dmctl test cmd: "query-status test-no-ignore-has-error" got=1 expected=1 wait process dm-master.test exit... process dm-master.test already exit ('create_task_with_precheck resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'name': u'test-has-ignore-without-error', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u'schema_of_shard_tables'], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-has-ignore-without-error" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later got=2 expected=2 delete_task_success wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success wait process dm-worker.test exit... process dm-worker.test already exit success success [Sun May 5 20:53:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:53:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 4-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=2 expected=2 check diff successfully [Sun May 5 20:53:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:53:16 CST 2024] <<<<<< finish DM-001 >>>>>> delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: TEST TASK WITH IGNORE CHECK ITEMS SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: DELETE TASK WITH STOPPED DOWNSTREAM ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) wait process dm-worker.test exit... ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}}) dmctl test cmd: "query-status test-no-shard" wait process dm-master.test exit... got=2 expected=2 tidb_server_num tidb-server alive got=1 expected=1 got=1 expected=1+ got=1 expected=1 got=1 expected=1 got=1 expected=1 --> start validator on the fly, validate from current syncer progress(gtid=false, relay=false) rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "query-status -s mysql-replica-01" got=3 expected=3 got=1 expected=1 got=1 expected=1 [Sun May 5 20:53:17 CST 2024] <<<<<< start DM-002 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process tidb-server exit... process tidb-server already exit check log contain failed 5-th time, retry later dmctl test cmd: "stop-task test" [Sun May 5 20:53:17 CST 2024] <<<<<< finish DM-104 optimistic >>>>>> ('delete_task_failed resp=', {u'error_code': 10001, u'error_msg': u'[code=10001:class=database:scope=downstream:level=high], Message: database driver error, RawCause: dial tcp 127.0.0.1:4000: connect: connection refused, Workaround: Please check the database connection and the database config in configuration file.'}) delete_task_success dmctl test cmd: "purge-relay --filename dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25-bin.000005 -s mysql-replica-01" dmctl test cmd: "query-status test" ('get_task_list resp=', {u'total': 0, u'data': []}) Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:18 CST 2024] <<<<<< test case online_ddl success! >>>>>> start running case: [foreign_key] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time kill dm-worker1 wait process dm-master.test exit... got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Sun May 5 20:53:18 CST 2024] <<<<<< finish DM-002 >>>>>> [Sun May 5 20:53:18 CST 2024] <<<<<< start DM-105 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker1 exit... wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 6-th time, retry later wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:53:19 CST 2024] <<<<<< start DM-003 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker1 exit... process dm-worker1 already exit run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner kill dm-worker2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker2 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/foreign_key/source1.yaml" check log contain failed 7-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-task.yaml --remove-meta" dmctl test cmd: "stop-task test" wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status -s mysql-replica-01" delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: DELETE TASK WITH STOPPED DOWNSTREAM SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: START TASK WITH CONDITION [Sun May 5 20:53:22 CST 2024] <<<<<< finish DM-105 optimistic >>>>>> wait process dm-master.test exit... got=2 expected=2 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) wait for rpc addr 127.0.0.1:8261 alive the 1-th time ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_no_source_meta'}}) dmctl test cmd: "query-status incremental_task_no_source_meta" got=2 expected=2 got=1 expected=1 got=2 expected=2 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check log contain failed 8-th time, retry later wait process dm-master.test exit... got=2 expected=2 ('start_task_failed resp=', {u'error_code': 20022, u'error_msg': u'[code=20022:class=config:scope=internal:level=medium], Message: mysql-instance(mysql-01) must set meta for task-mode incremental, Workaround: Please check the `meta` config in task configuration file.'}) check diff failed 2-th time, retry later check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success success finish running run safe mode recover case 3 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) [Sun May 5 20:53:23 CST 2024] <<<<<< start DM-106 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01', u'binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-58', u'binlog_name': u'dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001', u'binlog_pos': 11460}, {u'source_name': u'mysql-02', u'binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-54', u'binlog_name': u'dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001', u'binlog_pos': 11906}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_gtid'}}) dmctl test cmd: "query-status incremental_task_use_gtid" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "query-status incremental_task_use_gtid" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Sun May 5 20:53:23 CST 2024] <<<<<< finish DM-4215 pessimistic >>>>>> [Sun May 5 20:53:23 CST 2024] <<<<<< start DM-4215 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" clean source table dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "config export -p /tmp/configs" check no password in log 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=2 expected=2 run tidb sql failed 1-th time, retry later got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Sun May 5 20:53:26 CST 2024] <<<<<< finish DM-4215 optimistic >>>>>> check log contain failed 9-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:53:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... import prepare data start DM worker and master cluster [Sun May 5 20:53:26 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Sun May 5 20:53:26 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Sun May 5 20:53:26 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:53:27 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_start_time'}}) dmctl test cmd: "query-status incremental_task_use_start_time" got=2 expected=2 check diff failed at last check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... dmctl test cmd: "stop-task test" [Sun May 5 20:53:27 CST 2024] <<<<<< finish DM-106 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" [Sun May 5 20:53:27 CST 2024] <<<<<< finish DM-003 pessimistic >>>>>> check log contain failed 10-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:28 CST 2024] <<<<<< test case foreign_key success! >>>>>> start running case: [full_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 [Sun May 5 20:53:28 CST 2024] <<<<<< start DM-107 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:5043 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:5548 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" [Sun May 5 20:53:28 CST 2024] <<<<<< start DM-004 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time start to run safe mode case 4 [Sun May 5 20:53:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=3 expected=3 dmctl test cmd: "stop-task test" [Sun May 5 20:53:29 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Sun May 5 20:53:29 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 11-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sun May 5 20:53:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:6850 alter table handle_error.tb1 add column c varchar(20);" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:7521 alter table handle_error.tb1 add column c varchar(20);" start_task_with_condition success dmctl test cmd: "query-status incremental_task_use_start_time" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 12-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Sun May 5 20:53:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:53:32 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> [Sun May 5 20:53:32 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_start_time_after_create'}}) dmctl test cmd: "query-status incremental_task_use_start_time_after_create" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again [Sun May 5 20:53:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:53:32 CST 2024] <<<<<< finish DM-004 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" check log contain failed 13-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 [Sun May 5 20:53:33 CST 2024] <<<<<< finish DM-107 optimistic >>>>>> [Sun May 5 20:53:33 CST 2024] <<<<<< start DM-005 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:8915 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:9494 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /tmp/dm_test/full_mode/dm-task.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "stop-task test" [Sun May 5 20:53:34 CST 2024] <<<<<< start DM-108 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:53:34 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Sun May 5 20:53:34 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "config import -p /tmp/configs" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " use sync_diff_inspector to check increment data check diff successfully [Sun May 5 20:53:27 CST 2024] <<<<<< finish test_isolate_master_and_worker >>>>>> 5 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" check diff successfully [Sun May 5 20:53:35 CST 2024] <<<<<< finish DM-005 >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check log contain failed 14-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:10980 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase 1 dm-master alive wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:36 CST 2024] <<<<<< test case ha_cases3_1 success! >>>>>> start running case: [ha_master] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh... Verbose mode = false check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:11467 alter table handle_error.tb1 add column c varchar(20);" start_task_with_condition success dmctl test cmd: "query-status incremental_task_use_start_time_after_create" 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "operate-source show" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit start DM worker and master [Sun May 5 20:53:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Sun May 5 20:53:36 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> [Sun May 5 20:53:36 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> [Sun May 5 20:53:36 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> [Sun May 5 20:53:36 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 check diff successfully >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_operations passed 1 dm-master alive got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=false, relay=true) [Sun May 5 20:53:37 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:53:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time 2 dm-worker alive 0 dm-syncer alive ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01', u'binlog_gtid': u'96ae8793-0add-11ef-bf03-aa1057f7af0c:1-70', u'binlog_name': u'dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001', u'binlog_pos': 13696}, {u'source_name': u'mysql-02', u'binlog_gtid': u'9707fb65-0add-11ef-aee9-aa1057f7af0c:1-66', u'binlog_name': u'dm-it-b33cd601-076e-47ee-9c34-e5c1f3ce0751-m3v69-lz42n-bin.000001', u'binlog_pos': 14442}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_both_gtid_start_time'}}) dmctl test cmd: "query-status incremental_task_both_gtid_start_time" wait process dm-master.test exit... check log contain failed 15-th time, retry later got=2 expected=2 wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... check diff failed 2-th time, retry later [Sun May 5 20:53:38 CST 2024] <<<<<< start DM-4219 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 use sync_diff_inspector to check increment data check diff successfully [Sun May 5 20:53:39 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source kill dm-worker1 got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" wait process dm-worker.test exit... check log contain failed 16-th time, retry later got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker1 exit... wait process dm-worker.test exit... check diff failed 3-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit success success [Sun May 5 20:53:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:53:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:41 CST 2024] <<<<<< test case shardddl1 success! >>>>>> start running case: [shardddl1_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start_task_with_condition success dmctl test cmd: "query-status incremental_task_both_gtid_start_time" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit start worker3 [Sun May 5 20:53:41 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time check log contain failed 17-th time, retry later delete_task_success wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive wait process dm-worker.test exit... ('get_task_list resp=', {u'total': 0, u'data': []}) rpc addr 127.0.0.1:8461 is alive [Sun May 5 20:53:42 CST 2024] <<<<<< finish DM-4219 pessimistic >>>>>> [Sun May 5 20:53:42 CST 2024] <<<<<< start DM-4219 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" check diff failed at last dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker3" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8561 is alive rpc addr 127.0.0.1:8661 is alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Sun May 5 20:53:42 CST 2024] <<<<<< finish DM-108 optimistic >>>>>> [Sun May 5 20:53:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 kill dm-worker2 check log contain failed 18-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:53:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:53:44 CST 2024] <<<<<< start DM-109 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Sun May 5 20:53:44 CST 2024] <<<<<< finish DM-4219 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" wait process dm-worker2 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source1.yaml" [Sun May 5 20:53:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:53:45 CST 2024] <<<<<< start DM-4220 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 19-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 wait process dm-worker2 exit... process dm-worker2 already exit start worker4 [Sun May 5 20:53:45 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source2.yaml" dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner check diff failed 1-th time, retry later got=2 expected=2 kill dm-worker1 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" rpc addr 127.0.0.1:8263 is alive ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) [Sun May 5 20:53:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/full_mode/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:18262 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive query-status from all dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 [Sun May 5 20:53:47 CST 2024] <<<<<< start DM-SAME_DDL_TWICE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 20-th time, retry later wait process dm-worker1 exit... dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:16594" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:17605" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" check diff failed 1-th time, retry later check diff failed 2-th time, retry later kill dm-master1 and kill dm-master2 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:16787" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:17824" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:16787" wait process dm-worker1 exit... process dm-worker1 already exit [Sun May 5 20:53:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:17824" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "config source mysql-replica-01" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-master1 exit... got=1 expected=1 got=1 expected=1 run tidb sql failed 1-th time, retry later check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success success finish running run safe mode recover case 4 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_upstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/full_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/full_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master1 exit... process dm-master1 already exit check log contain failed 21-th time, retry later dmctl test cmd: "query-status test" check diff failed 3-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" wait process dm-master.test exit... wait process dm-master2 exit... got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_no_duration_but_error'}}) dmctl test cmd: "query-status incremental_task_no_duration_but_error" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_no_duration_but_error" got=2 expected=2 start_task_with_condition success run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:16787" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:17824" dmctl test cmd: "binlog revert test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:17824" dmctl test cmd: "binlog revert test" dmctl test cmd: "stop-task test" wait process dm-master2 exit... process dm-master2 already exit check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:53:51 CST 2024] <<<<<< finish DM-SAME_DDL_TWICE pessimistic >>>>>> [Sun May 5 20:53:51 CST 2024] <<<<<< start DM-BINARY_COLUMN >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-sharding.yaml --remove-meta" check log contain failed 22-th time, retry later [Sun May 5 20:53:52 CST 2024] <<<<<< finish DM-4220 pessimistic >>>>>> [Sun May 5 20:53:52 CST 2024] <<<<<< start DM-4220 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" master4 [Sun May 5 20:53:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed at last dmctl test cmd: "stop-task test" [Sun May 5 20:53:52 CST 2024] <<<<<< finish DM-109 optimistic >>>>>> wait process dm-worker.test exit... dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check diff successfully check diff successfully check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 2-th time wait process dm-worker.test exit... delete_task_success dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success [Sun May 5 20:53:53 CST 2024] <<<<<< start DM-110 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 23-th time, retry later wait process dm-worker.test exit... ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: STOP TASK WITH CONDITION [Sun May 5 20:53:53 CST 2024] <<<<<< finish DM-BINARY_COLUMN >>>>>> [Sun May 5 20:53:53 CST 2024] <<<<<< start DM-006 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:19218" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:20496" ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) wait for rpc addr 127.0.0.1:8262 alive the 3-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) [Sun May 5 20:53:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:19411" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:20715" dmctl test cmd: "query-status test" ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:20715" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:53:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait for rpc addr 127.0.0.1:8262 alive the 4-th time got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... check log contain failed 24-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 1-th time, retry later wait for rpc addr 127.0.0.1:8262 alive the 5-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:53:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:53:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source1.yaml" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=true, relay=false) rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Sun May 5 20:53:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 25-th time, retry later wait for rpc addr 127.0.0.1:8262 alive the 6-th time check diff failed 2-th time, retry later rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:53:57 CST 2024] <<<<<< finish DM-006 >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" [Sun May 5 20:53:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:20715" dmctl test cmd: "binlog revert test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:20715" dmctl test cmd: "binlog revert test" dmctl test cmd: "stop-task test" [Sun May 5 20:53:58 CST 2024] <<<<<< finish DM-4220 optimistic >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 7-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" [Sun May 5 20:53:58 CST 2024] <<<<<< start DM-007 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" rpc addr 127.0.0.1:8263 is alive ('create_noshard_task_success resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test_wait_time_on_stop'}}) dmctl test cmd: "query-status test_wait_time_on_stop" got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 [Sun May 5 20:53:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test_wait_time_on_stop" got=2 expected=2 check log contain failed 26-th time, retry later rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:53:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... check diff failed 3-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task.yaml --remove-meta" check diff successfully error check [Sun May 5 20:53:59 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task-safe-mode-duration.yaml --remove-meta" wait for rpc addr 127.0.0.1:8263 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:54:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" check diff successfully got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8263 alive the 3-th time wait process dm-worker.test exit... check diff failed at last dmctl test cmd: "stop-task test" dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:21842" dmctl test cmd: "query-status test" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" [Sun May 5 20:54:01 CST 2024] <<<<<< finish DM-110 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:23387" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" check log contain failed 27-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker.test exit... dmctl test cmd: "stop-task test" dmctl test cmd: "query-status -s mysql-replica-02" [Sun May 5 20:54:02 CST 2024] <<<<<< start DM-111 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:02 CST 2024] <<<<<< finish DM-007 >>>>>> [Sun May 5 20:54:02 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Sun May 5 20:54:02 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8263 alive the 4-th time got=2 expected=2 got=1 expected=1 [Sun May 5 20:54:02 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) [Sun May 5 20:54:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:54:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:54:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:54:03 CST 2024] <<<<<< start DM-008 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: REVERSE HTTPS 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8263 alive the 5-th time rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3" check log contain failed 28-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8263 alive the 6-th time dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:23811" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:25545" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -n worker3" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later [Sun May 5 20:54:05 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> [Sun May 5 20:54:05 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8263 alive the 7-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:54:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 29-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " check diff failed 2-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive start dm-master1 and dm-master2 again [Sun May 5 20:54:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Sun May 5 20:54:06 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:26038" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:27703" check log contain failed 30-th time, retry later check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:07 CST 2024] <<<<<< finish DM-008 >>>>>> rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "config source mysql-replica-01" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... [Sun May 5 20:54:08 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Sun May 5 20:54:08 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... [Sun May 5 20:54:08 CST 2024] <<<<<< start DM-009 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check diff failed 3-th time, retry later check diff successfully finish running safe mode duration case 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:54:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Sun May 5 20:54:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:28265" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:29861" wait process dm-master.test exit... dmctl test cmd: "list-member --master" got=5 expected=5 [Sun May 5 20:54:10 CST 2024] <<<<<< start test_evict_leader >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" check diff failed at last dmctl test cmd: "stop-task test" [Sun May 5 20:54:10 CST 2024] <<<<<< finish DM-111 optimistic >>>>>> leader is master4 dmctl test cmd: "operate-leader evict" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 14945 0 --:--:-- --:--:-- --:--:-- 16500 [Sun May 5 20:54:11 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> new leader is master3 wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> [Sun May 5 20:54:11 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time leader is master3 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 15691 0 --:--:-- --:--:-- --:--:-- 16500 dmctl test cmd: "validation status test" new leader is master5 wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task-2.yaml --remove-meta" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again leader is master5 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 17386 0 --:--:-- --:--:-- --:--:-- 33000 [Sun May 5 20:54:11 CST 2024] <<<<<< start DM-112 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check diff successfully [Sun May 5 20:54:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... new leader is master1 leader is master1 dmctl test cmd: "operate-leader evict" rpc addr 127.0.0.1:8262 is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 15507 0 --:--:-- --:--:-- --:--:-- 16500 [Sun May 5 20:54:12 CST 2024] <<<<<< start DM-4201 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" rpc addr 127.0.0.1:8264 is alive [Sun May 5 20:54:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/openapi/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time new leader is master2 leader is master2 dmctl test cmd: "operate-leader evict" dmctl test cmd: "query-status test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 16168 0 --:--:-- --:--:-- --:--:-- 33000 cancel evict leader on master1, and master1 will be the leader dmctl test cmd: "operate-leader cancel-evict" cancel evict leader twice, and test cancel evict leader from http interface % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 20846 0 --:--:-- --:--:-- --:--:-- 33000 wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive leader is master1 cancel evict leader on all masters cancel master port 8361 dmctl test cmd: "operate-leader cancel-evict" cancel master port 8461 dmctl test cmd: "operate-leader cancel-evict" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" cancel master port 8561 dmctl test cmd: "operate-leader cancel-evict" cancel master port 8661 dmctl test cmd: "operate-leader cancel-evict" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status -s mysql-replica-01" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:13 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/openapi/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 [Sun May 5 20:54:13 CST 2024] <<<<<< finish DM-009 >>>>>> [Sun May 5 20:54:13 CST 2024] <<<<<< finish test_evict_leader >>>>>> [Sun May 5 20:54:13 CST 2024] <<<<<< start test_list_member_command >>>>>> got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-02" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member --worker" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:30234" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> >>>>>>>>>>>>>>>>>>>>>>>>>>test test_restart_relay_status passed 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=2 expected=2 dmctl test cmd: "stop-task test" [Sun May 5 20:54:14 CST 2024] <<<<<< start DM-010 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" current leader is master1 dmctl test cmd: "list-member --master" [Sun May 5 20:54:14 CST 2024] <<<<<< finish DM-4201 >>>>>> wait process dm-master.test exit... got=5 expected=5 kill leader master1 wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit wait process master1 exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=true, relay=true) got=1 expected=1 check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process master1 exit... process master1 already exit wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" current leader is master2 dmctl test cmd: "list-member --master" got=4 expected=4 kill leader master2 check diff failed 3-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:18 CST 2024] <<<<<< finish DM-010 >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process master2 exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:54:19 CST 2024] <<<<<< start DM-011 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit check diff failed at last dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:19 CST 2024] <<<<<< test case handle_error_2 success! >>>>>> start running case: [handle_error_3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:54:20 CST 2024] <<<<<< finish DM-112 optimistic >>>>>> wait process master2 exit... process master2 already exit [Sun May 5 20:54:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully wait for port 8262 offline the 1-th time wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:54:21 CST 2024] <<<<<< start DM-113 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:21 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time dmctl test cmd: "stop-task test" wait for port 8262 offline the 2-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:54:21 CST 2024] <<<<<< finish DM-011 >>>>>> wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:21 CST 2024] <<<<<< test case duplicate_event success! >>>>>> start running case: [expression_filter] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source1.yaml" [Sun May 5 20:54:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker2.toml >>>>>> [Sun May 5 20:54:22 CST 2024] <<<<<< start DM-012 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8361 is alive wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:54:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:54:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task.yaml --remove-meta" current leader is master5 dmctl test cmd: "list-member --master" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source2.yaml" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=5 expected=5 wait for rpc addr 127.0.0.1:8361 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8361 is alive [Sun May 5 20:54:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/openapi/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time current leader is master5 kill master1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" [Sun May 5 20:54:24 CST 2024] <<<<<< finish DM-012 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/full_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/full_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sun May 5 20:54:24 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master1 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:54:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:25 CST 2024] <<<<<< start DM-013 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/openapi/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:54:25 CST 2024] <<<<<< finish DM-113 pessimistic >>>>>> [Sun May 5 20:54:25 CST 2024] <<<<<< start DM-113 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml " kill dm-worker1 wait process dm-master1 exit... process dm-master1 already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive ('create_source_success_https resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_success_https resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:1559 alter table handle_error.tb1 add column d varchar(20);" ('list_source_with_reverse_https resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:1735 alter table handle_error.tb1 add column d varchar(20);" wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:27 CST 2024] <<<<<< finish DM-013 >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 [Sun May 5 20:54:27 CST 2024] <<<<<< finish DM-113 optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "stop-task test" wait process dm-worker1 exit... process dm-worker1 already exit [Sun May 5 20:54:27 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Sun May 5 20:54:27 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... [Sun May 5 20:54:28 CST 2024] <<<<<< start DM-014 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" wait process tidb-server exit... process tidb-server already exit [Sun May 5 20:54:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:54:28 CST 2024] <<<<<< start DM-114 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... check sync diff after set SafeModeInitPhaseSeconds failpoint check diff successfully wait for one of DM-worker offine failed, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "validation status test" got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" ls: cannot access /tmp/dm_test/expression_filter/worker1/schema-tracker*: No such file or directory schema tracker path has been cleaned 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:3856 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit wait for one of DM-worker offine failed, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:29 CST 2024] <<<<<< test case full_mode success! >>>>>> start running case: [gbk] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-master.toml >>>>>> got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:4247 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... DM-worker1 is sharding lock owner and detects it offline [Sun May 5 20:54:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:54:30 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> [Sun May 5 20:54:30 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... [Sun May 5 20:54:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_downstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member --master" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Sun May 5 20:54:31 CST 2024] <<<<<< finish DM-014 >>>>>> got=5 expected=5 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time current leader is master5 kill master2 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:6411 alter table handle_error.tb1 add column d varchar(20);" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Sun May 5 20:54:32 CST 2024] <<<<<< finish DM-114 pessimistic >>>>>> [Sun May 5 20:54:32 CST 2024] <<<<<< start DM-114 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:6759 alter table handle_error.tb1 add column d varchar(20);" wait process dm-worker.test exit... [Sun May 5 20:54:32 CST 2024] <<<<<< start DM-015 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master2 exit... [Sun May 5 20:54:33 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Sun May 5 20:54:33 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:33 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gbk/source1.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/gbk/source2.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master2 exit... process dm-master2 already exit dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time < min mysql binlog pos(gtid=false) dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:8966 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:9271 alter table handle_error.tb1 add column d varchar(20);" prepare data start task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:36 CST 2024] <<<<<< finish DM-114 optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sun May 5 20:54:36 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> [Sun May 5 20:54:36 CST 2024] <<<<<< finish DM-015 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task.yaml " check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully prepare incremental data check incremental phase check diff successfully ERROR 1146 (42S02) at line 1: Table 'gbk.ddl1' doesn't exist run tidb sql failed 1-th time, retry later [Sun May 5 20:54:37 CST 2024] <<<<<< start DM-115 optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:54:37 CST 2024] <<<<<< start DM-016 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" [Sun May 5 20:54:37 CST 2024] <<<<<< start DM-4210 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later ERROR 1146 (42S02) at line 1: Table 'gbk.ddl2_copy' doesn't exist run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "binlog replace test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:11239 alter table handle_error.tb1 add column d int;alter table handle_error.tb1 add unique(d);" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column e int unique;" dmctl test cmd: "query-status test" wait process dm-worker.test exit... [Sun May 5 20:54:39 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> got=1 expected=1 wait process dm-worker.test exit... dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8361 alive the 1-th time wait process dm-worker.test exit... got=2 expected=2 run tidb sql failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=0 expected=1 command: query-status -s mysql-replica-01 "relayCatchUpMaster": true count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... rpc addr 127.0.0.1:8361 is alive dmctl test cmd: "list-member --master" got=5 expected=5 dmctl test cmd: "stop-task test" current leader is master5 kill master3 check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:41 CST 2024] <<<<<< finish DM-115 optimistic >>>>>> [Sun May 5 20:54:41 CST 2024] <<<<<< finish DM-016 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> ERROR 1146 (42S02) at line 1: Table 'gbk.ddl3' doesn't exist run tidb sql failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:41 CST 2024] <<<<<< test case safe_mode success! >>>>>> start running case: [sequence_safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master3 exit... [Sun May 5 20:54:42 CST 2024] <<<<<< start DM-116 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" [Sun May 5 20:54:42 CST 2024] <<<<<< start DM-017 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" [Sun May 5 20:54:42 CST 2024] <<<<<< finish DM-4210 >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master3 exit... process dm-master3 already exit got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_kill_dump_connection passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Sun May 5 20:54:43 CST 2024] <<<<<< start DM-4193 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive ERROR 1049 (42000) at line 1: Unknown database 'gbk3' run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8361 alive the 2-th time dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8361 is alive [Sun May 5 20:54:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Sun May 5 20:54:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:13027" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:12152" wait process dm-master.test exit... process dm-master.test already exit ERROR 1146 (42S02) at line 1: Table 'gbk.ddl4' doesn't exist run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:13027" dmctl test cmd: "stop-task test" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:12152" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:13413" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog revert test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:13413" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:13413" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source2.yaml" [Sun May 5 20:54:45 CST 2024] <<<<<< finish DM-017 >>>>>> dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:12590" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:46 CST 2024] <<<<<< finish DM-116 pessimistic >>>>>> [Sun May 5 20:54:46 CST 2024] <<<<<< start DM-116 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: REVERSE HTTPS ('list_master_success resp=', {u'total': 2, u'data': [{u'name': u'master1', u'addr': u'http://127.0.0.1:8291', u'alive': True, u'leader': True}, {u'name': u'master2', u'addr': u'http://127.0.0.1:8292', u'alive': True, u'leader': False}]}) ('list_worker_success resp=', {u'total': 2, u'data': [{u'bound_source_name': u'', u'bound_stage': u'free', u'addr': u'127.0.0.1:8262', u'name': u'worker1'}, {u'bound_source_name': u'', u'bound_stage': u'free', u'addr': u'127.0.0.1:8263', u'name': u'worker2'}]}) got=3 expected=3 run tidb sql failed 1-th time, retry later [Sun May 5 20:54:46 CST 2024] <<<<<< start DM-018 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" delete_master_with_retry_success ('list_master_success resp=', {u'total': 1, u'data': [{u'name': u'master1', u'addr': u'http://127.0.0.1:8291', u'alive': True, u'leader': True}]}) ('delete_worker_failed resp=', {u'error_code': 46005, u'error_msg': u'[code=46005:class=scheduler:scope=internal:level=medium], Message: dm-worker with name worker1 is still online, Workaround: Please shut it down first.'}) prepare data for invalid connection test check log contain failed 1-th time, retry later dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff failed 1-th time, retry later [Sun May 5 20:54:48 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:48 CST 2024] <<<<<< test case expression_filter success! >>>>>> start running case: [extend_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: sql [Sun May 5 20:54:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... check diff successfully wait for port 8262 offline the 1-th time dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:48 CST 2024] <<<<<< test case new_relay success! >>>>>> start running case: [all_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:48 CST 2024] <<<<<< start test_expression_filter >>>>>> [Sun May 5 20:54:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:54:49 CST 2024] <<<<<< finish DM-4193 pessimistic >>>>>> [Sun May 5 20:54:49 CST 2024] <<<<<< start DM-4193 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8461 is alive dmctl test cmd: "list-member --master" got=5 expected=5 wait for port 8262 offline the 2-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time current leader is master5 kill master4 wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-worker.test exit... check diff failed 2-th time, retry later [Sun May 5 20:54:50 CST 2024] <<<<<< finish DM-018 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master4 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:15189" [Sun May 5 20:54:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:54:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:14529" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:54:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:54:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:15189" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:14529" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" wait process dm-master4 exit... process dm-master4 already exit dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:15575" dmctl test cmd: "binlog revert test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:15575" [Sun May 5 20:54:51 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:15575" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:14967" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:54:52 CST 2024] <<<<<< finish DM-116 optimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time < min mysql binlog pos(gtid=true) dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:54:53 CST 2024] <<<<<< finish DM-4193 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive start test invalid connection with status running rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" [Sun May 5 20:54:53 CST 2024] <<<<<< start DM-117 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:54:53 CST 2024] <<<<<< finish DM-019 >>>>>> [Sun May 5 20:54:53 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" check count check diff successfully check test invalid connection with status running successfully dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-worker.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check sync diff after reset failpoint [Sun May 5 20:54:54 CST 2024] <<<<<< start DM-4230 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" check diff successfully wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status test" [Sun May 5 20:54:54 CST 2024] <<<<<< finish DM-019 >>>>>> wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl3_1/schema.sql -s mysql-replica-01" dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column d int unique;" [Sun May 5 20:54:55 CST 2024] <<<<<< finish DM-117 optimistic >>>>>> wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog revert test" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit restart DM-worker after set SequenceShardSyncedExecutionExit failpoint [Sun May 5 20:54:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Sun May 5 20:54:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" check diff successfully check diff failed 1-th time, retry later wait process dm-worker.test exit... [Sun May 5 20:54:55 CST 2024] <<<<<< start DM-020 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "stop-task test" [Sun May 5 20:54:56 CST 2024] <<<<<< finish DM-4230 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:54:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... [Sun May 5 20:54:56 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time [Sun May 5 20:54:56 CST 2024] <<<<<< start DM-118 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive got=1 expected=1 run tidb sql failed 1-th time, retry later [Sun May 5 20:54:57 CST 2024] <<<<<< start DM-4177 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:54:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "list-member --master" dmctl test cmd: "query-status test" wait process dm-master.test exit... got=5 expected=5 got=2 expected=2 check diff failed 1-th time, retry later current leader is master5 dmctl test cmd: "list-member --worker --name=worker1,worker2" rpc addr 127.0.0.1:8263 is alive start test invalid connection with status queueing run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit delete_worker_with_retry_success ('list_worker_success resp=', {u'total': 1, u'data': [{u'bound_source_name': u'', u'bound_stage': u'offline', u'addr': u'127.0.0.1:8263', u'name': u'worker2'}]}) 2 dm-master alive 0 dm-worker alive 0 dm-syncer alive got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source1.yaml" check diff failed 2-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:54:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "list-member --worker" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "list-member" wait for one of DM-worker offine failed, retry later got=1 expected=1 dmctl test cmd: "binlog skip wrong-task" dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" dmctl test cmd: "binlog skip" dmctl test cmd: "binlog skip test -b dm-it-64f08141-f09f-467f-bf00-09430931b325-s2sqf-dnlqt-bin|000001.000001:19397 -s wrong-source" dmctl test cmd: "binlog skip test -b mysql-bin|1111 -s wrong-source" [Sun May 5 20:54:59 CST 2024] <<<<<< finish DM-020 >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int;"" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "binlog skip test -b mysql-bin.000000:00000" dmctl test cmd: "binlog aaa test" wait process dm-master.test exit... dmctl test cmd: "stop-task test" wait for one of DM-worker offine failed, retry later check diff failed 3-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:55:00 CST 2024] <<<<<< finish DM-118 pessimistic >>>>>> [Sun May 5 20:55:00 CST 2024] <<<<<< start DM-118 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:55:00 CST 2024] <<<<<< finish DM-4177 >>>>>> dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int" dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" wait process dm-worker.test exit... [Sun May 5 20:55:00 CST 2024] <<<<<< start DM-021 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog revert test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check count check diff successfully check test invalid connection with status queueing successfully wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:01 CST 2024] <<<<<< test case openapi success! >>>>>> start running case: [s3_dumpling_lightning] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh... Verbose mode = false DM-worker1 is sharding lock owner and detects it offline [Sun May 5 20:55:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit Starting PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:2379; Connection refused 2024-05-05 20:55:01.597566 W | pkg/fileutil: check file permission: directory "/tmp/dm_test/s3_dumpling_lightning.downstream/pd" exist, but the permission is "drwxr-xr-x". The recommended permission is "-rwx------" to prevent possible unprivileged access to the data. wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: loader dmctl test cmd: "query-status test" [Sun May 5 20:55:01 CST 2024] <<<<<< start DM-4231 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" wait process dm-worker.test exit... [Sun May 5 20:55:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check log contain failed 1-th time, retry later dmctl test cmd: "list-member --name=worker1,worker2" got=2 expected=2 kill worker1 [Sun May 5 20:55:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive check diff failed at last dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int; alter table handle_error.tb1 add column d int unique;"" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" [Sun May 5 20:55:04 CST 2024] <<<<<< finish DM-4231 >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --name=worker1,worker2" got=1 expected=1 got=1 expected=1 kill worker2 [Sun May 5 20:55:04 CST 2024] <<<<<< finish DM-021 >>>>>> dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:55:04 CST 2024] <<<<<< finish DM-118 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive start test invalid connection with status none check count % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 61565 0 --:--:-- --:--:-- --:--:-- 104k check diff successfully check test invalid connection with status none successfully rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker2 exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sun May 5 20:55:05 CST 2024] <<<<<< start DM-022 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" wait process dm-worker.test exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member" got=2 expected=2 [Sun May 5 20:55:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:07 CST 2024] <<<<<< finish test_expression_filter >>>>>> [Sun May 5 20:55:07 CST 2024] <<<<<< start test_json_expression >>>>>> [Sun May 5 20:55:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" wait process dm-master.test exit... dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Sun May 5 20:55:06 CST 2024] <<<<<< finish test_kill_worker_in_sync >>>>>> 3 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... [Sun May 5 20:55:09 CST 2024] <<<<<< finish DM-022 >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config master master1" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "config master master2" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit got=1 expected=1 [Sun May 5 20:55:09 CST 2024] <<<<<< finish test_list_member_command >>>>>> rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... dmctl test cmd: "start-relay -s mysql-replica-01 worker1" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 101k 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused rpc addr 127.0.0.1:8263 is alive start test inserting data after invalid connection check count dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time which is in range of mysql binlog(gtid=false) wait process dm-master.test exit... wait process dm-worker.test exit... [Sun May 5 20:55:10 CST 2024] <<<<<< start DM-023 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully check test inserting data after invalid connection successfully wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:10 CST 2024] <<<<<< test case handle_error_3 success! >>>>>> start running case: [adjust_gtid] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:10 CST 2024] <<<<<< test case shardddl3_1 success! >>>>>> start running case: [shardddl4] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:55:11 CST 2024] <<<<<< finish DM-023 >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" wait process dm-worker.test exit... use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file kill dm-master1 and kill dm-master2 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:12 CST 2024] <<<<<< test case sequence_safe_mode success! >>>>>> start running case: [lightning_load_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM master, workers and sources [Sun May 5 20:55:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source1.yaml" [Sun May 5 20:55:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:55:12 CST 2024] <<<<<< start DM-026 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source1.yaml" wait process dm-worker.test exit... wait process dm-master1 exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... [Sun May 5 20:55:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master1 exit... process dm-master1 already exit wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive start test adding UNIQUE on column with duplicate data check cancelled error dmctl test cmd: "query-status gbk" got=0 expected=1 command: query-status gbk origin SQL: \[ALTER TABLE gbk.invalid_conn_test1 ADD UNIQUE(i)\]: DDL ALTER TABLE `gbk`.`invalid_conn_test1` ADD UNIQUE(`i`) executed in background and met error count: 0 != expected: 1, failed the 0-th time, will retry again check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:55:14 CST 2024] <<<<<< finish DM-026 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:14 CST 2024] <<<<<< test case extend_column success! >>>>>> start running case: [fake_rotate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_load_task/source1.yaml" [Sun May 5 20:55:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker2.toml >>>>>> dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml --remove-meta" wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source2.yaml" wait process dm-master2 exit... wait process dm-master.test exit... [Sun May 5 20:55:15 CST 2024] <<<<<< start DM-027 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 123 100 123 0 0 247k 0 --:--:-- --:--:-- --:--:-- 120k {"connections":0,"version":"8.0.11-TiDB-v8.2.0-alpha-79-g600b2ed4bf","git_hash":"600b2ed4bf0aa38224a1c4c4c68831820735515c"}process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_load_task/source2.yaml" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:16 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:55:16 CST 2024] <<<<<< start DM-119 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/fake_rotate_event/source1.yaml" wait process dm-master.test exit... wait process dm-master2 exit... process dm-master2 already exit wait and check task running HTTP 127.0.0.1:8461/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" got=3 expected=3 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-task.yaml --remove-meta" Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide got=1 expected=1 dmctl test cmd: "resume-task gbk" got=3 expected=3 check test adding UNIQUE on column with duplicate data successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:55:17 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC98F0AFFCD397 < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 12:55:17 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-master.test already exit Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-worker.test already exit [Sun May 5 20:55:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> [Sun May 5 20:55:17 CST 2024] <<<<<< finish DM-027 >>>>>> wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check diff successfully kill dm-worker rpc addr 127.0.0.1:8264 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:55:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:55:18 CST 2024] <<<<<< start DM-028 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time use sync_diff_inspector to check data now! check diff successfully dmctl test cmd: "offline-member --master --name master1" dmctl test cmd: "offline-member --master --name master2" kill dm-master3 wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" wait process dm-worker1 exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task2.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" wait process dm-worker.test exit... [Sun May 5 20:55:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 2-th time, retry later got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:55:20 CST 2024] <<<<<< finish DM-028 >>>>>> wait process dm-master3 exit... wait process dm-master.test exit... wait process dm-worker1 exit... process dm-worker1 already exit [Sun May 5 20:55:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status load_task1" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:20 CST 2024] <<<<<< finish test_json_expression >>>>>> [Sun May 5 20:55:20 CST 2024] <<<<<< start test_fail_job_between_event >>>>>> [Sun May 5 20:55:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 got=1 expected=1 test worker restart dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8263 is alive start test invalid connection with status running (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "query-status test" [Sun May 5 20:55:21 CST 2024] <<<<<< start DM-030 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master3 exit... process dm-master3 already exit got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker1 exit... wait process dm-master.test exit... check diff failed 3-th time, retry later dmctl test cmd: "query-status test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 [Sun May 5 20:55:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check count 2 check diff successfully check test invalid connection with status running (multi-schema change) successfully dmctl test cmd: "query-status test" got=3 expected=3 [Sun May 5 20:55:23 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit prepare source data rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Sun May 5 20:55:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check diff failed 4-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" wait process dm-worker.test exit... rpc addr 127.0.0.1:8461 is alive got=1 expected=1 got=1 expected=1 test_transfer_two_sources dmctl test cmd: "validation start --start-time '2024-05-05 14:55:24' test" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:55:25 CST 2024] <<<<<< finish DM-030 pessimistic >>>>>> [Sun May 5 20:55:25 CST 2024] <<<<<< start DM-030 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-worker.test exit... wait process dm-worker2 exit... check task result run tidb sql failed 1-th time, retry later check diff failed 5-th time, retry later dmctl test cmd: "query-status test" [Sun May 5 20:55:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:26 CST 2024] <<<<<< test case fake_rotate_event success! >>>>>> [Pipeline] } wait process dm-worker2 exit... Cache not saved (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Sun May 5 20:55:27 CST 2024] <<<<<< finish DM-030 optimistic >>>>>> [Pipeline] } [Pipeline] // withCredentials process dm-worker2 already exit dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] // container rpc addr 127.0.0.1:8263 is alive start test invalid connection with status queueing (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task3.yaml --remove-meta" [Pipeline] } wait process dm-master.test exit... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml" wait process dm-master.test exit... [Sun May 5 20:55:28 CST 2024] <<<<<< start DM-031 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "query-status load_task3" got=1 expected=1 [Sun May 5 20:55:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 2-th time, retry later check diff failed 6-th time, retry later got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again check count 2 check diff successfully check test invalid connection with status queueing (multi-schema change) successfully wait for port 8262 offline the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member -w -n worker2" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" got=1 expected=1 [Sun May 5 20:55:29 CST 2024] <<<<<< finish DM-031 pessimistic >>>>>> [Sun May 5 20:55:29 CST 2024] <<<<<< start DM-031 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:55:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... run tidb sql failed 3-th time, retry later check diff failed 7-th time, retry later wait process dm-worker1 exit... [Sun May 5 20:55:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time which is in range of mysql binlog(gtid=true) rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "pause-task test" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time 4 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member -w -n worker2" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task4.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... run tidb sql failed 1-th time, retry later check diff failed 8-th time, retry later dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:55:33 CST 2024] <<<<<< finish DM-031 optimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "query-status load_task4" got=1 expected=1 [Sun May 5 20:55:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive start test invalid connection with status none (multi-schema change) check count 1 check count 2 check diff successfully check test invalid connection with status none (multi-schema change) successfully wait process dm-worker.test exit... [Sun May 5 20:55:34 CST 2024] <<<<<< start DM-032 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member -w -n worker1" wait process dm-master.test exit... run tidb sql failed 2-th time, retry later got=1 expected=1 wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... check diff failed 9-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker3 exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:35 CST 2024] <<<<<< finish test_fail_job_between_event >>>>>> [Sun May 5 20:55:35 CST 2024] <<<<<< start test_session_config >>>>>> [Sun May 5 20:55:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:55:36 CST 2024] <<<<<< finish DM-032 optimistic >>>>>> run tidb sql failed 3-th time, retry later wait process dm-worker3 exit... process dm-worker3 already exit wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 10-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Sun May 5 20:55:37 CST 2024] <<<<<< start DM-033 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" [Sun May 5 20:55:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:37 CST 2024] <<<<<< test case adjust_gtid success! >>>>>> start running case: [async_checkpoint_flush] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:55:39 CST 2024] <<<<<< finish DM-119 pessimistic >>>>>> [Sun May 5 20:55:39 CST 2024] <<<<<< start DM-119 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start test inserting data after invalid connection (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 1-th time, will retry again wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 14443 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/async_checkpoint_flush/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:40 CST 2024] <<<<<< test case ha_master success! >>>>>> start running case: [tracker_ignored_ddl] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later [Sun May 5 20:55:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check count 2 check diff successfully check test inserting data after invalid connection (multi-schema change) successfully check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:55:41 CST 2024] <<<<<< finish DM-033 pessimistic >>>>>> got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 2-th time, will retry again wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" check diff successfully PID of insert_data is 27037 * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC98F646EE9AAF < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 12:55:41 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" [Sun May 5 20:55:42 CST 2024] <<<<<< start DM-034 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tracker_ignored_ddl/source1.yaml" wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-task.yaml " wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 [Sun May 5 20:55:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 2-th time, retry later check diff successfully dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 3-th time, will retry again rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully increment1 check success [Sun May 5 20:55:43 CST 2024] <<<<<< finish DM-034 pessimistic >>>>>> [Sun May 5 20:55:43 CST 2024] <<<<<< start DM-034 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 rpc addr 127.0.0.1:8263 is alive start test adding UNIQUE on column with duplicate data (multi-schema change) check cancelled error dmctl test cmd: "query-status gbk" got=0 expected=1 command: query-status gbk origin SQL: \[ALTER TABLE gbk.invalid_conn_test1 ADD UNIQUE(k), ADD UNIQUE(m)\]: DDL ALTER TABLE `gbk`.`invalid_conn_test1` ADD UNIQUE(`k`) executed in background and met error count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 4-th time, will retry again wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... [Sun May 5 20:55:45 CST 2024] <<<<<< finish DM-034 optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "resume-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 increment2 check success dmctl test cmd: "stop-relay -s mysql-replica-01" wait process dm-worker.test exit... check diff failed 4-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:55:46 CST 2024] <<<<<< start DM-035 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "operate-source stop /tmp/dm_test/tracker_ignored_ddl/source1.yaml" [Sun May 5 20:55:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-master.toml >>>>>> wait process dm-master.test exit... got=1 expected=1 check test adding UNIQUE on column with duplicate data (multi-schema change) successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "validation start --start-time '2024-05-05 14:55:46' test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-worker.toml >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tracker_ignored_ddl/source1.yaml" got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -w -n worker2" wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status" got=3 expected=3 got=1 expected=1 [Sun May 5 20:55:47 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker3.toml >>>>>> wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-task.yaml " wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:48 CST 2024] <<<<<< finish test_session_config >>>>>> [Sun May 5 20:55:48 CST 2024] <<<<<< start test_query_timeout >>>>>> [Sun May 5 20:55:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:55:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status" got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 0-th time, will retry again check diff failed 5-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff successfully increment1 check success rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "stop-task test" [Sun May 5 20:55:50 CST 2024] <<<<<< finish DM-035 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:55:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:55:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 6-th time, retry later got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again wait process dm-master.test exit... wait process dm-worker.test exit... got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 1-th time, will retry again 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "resume-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 increment2 check success dmctl test cmd: "stop-relay -s mysql-replica-01" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:52 CST 2024] <<<<<< test case gbk success! >>>>>> start running case: [gtid] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/run.sh... Verbose mode = false dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "operate-source stop /tmp/dm_test/tracker_ignored_ddl/source1.yaml" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff failed 7-th time, retry later got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time > max mysql binlog pos wait process dm-master.test exit... dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 2-th time, will retry again rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source1.yaml" wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:55:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... check task result run tidb sql failed 1-th time, retry later check diff failed 8-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source2.yaml" got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 3-th time, will retry again wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:56 CST 2024] <<<<<< test case shardddl1_1 success! >>>>>> start running case: [shardddl2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh... Verbose mode = false wait process dm-worker.test exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... check diff failed 9-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:55:57 CST 2024] <<<<<< test case tracker_ignored_ddl success! >>>>>> start running case: [validator_checkpoint] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh... Verbose mode = false --> check persist checkpoint and data with 2 source 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit run tidb sql failed 2-th time, retry later got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 4-th time, will retry again rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:55:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:55:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit check diff failed 10-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source1.yaml" process dm-syncer.test already exit [Sun May 5 20:55:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 5-th time, will retry again run tidb sql failed 3-th time, retry later [Sun May 5 20:56:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... run tidb sql failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" [Sun May 5 20:56:01 CST 2024] <<<<<< finish DM-119 optimistic >>>>>> wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml" got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 6-th time, will retry again dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" wait process dm-master.test exit... [Sun May 5 20:56:02 CST 2024] <<<<<< start DM-120 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" [Sun May 5 20:56:02 CST 2024] <<<<<< start DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:56:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... run tidb sql failed 2-th time, retry later dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta" got=2 expected=2 restart dm-worker 1 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit check diff failed 1-th time, retry later [Sun May 5 20:56:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-worker1 exit... dmctl test cmd: "config source mysql-replica-01" dmctl test cmd: ""unit": "Sync"" dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 got=1 expected=1 run tidb sql failed 3-th time, retry later got=0 expected=1 command: validation status test processedRowsStatus": "insert\/update\/delete: 3\/1\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=4 expected=4 check diff successfully check diff successfully check diff successfully wait process dm-worker1 exit... process dm-worker1 already exit check diff successfully 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... [Sun May 5 20:56:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source1.yaml" [Sun May 5 20:56:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-master.test exit... [Sun May 5 20:56:06 CST 2024] <<<<<< finish DM-120 pessimistic >>>>>> [Sun May 5 20:56:06 CST 2024] <<<<<< start DM-120 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive check log contain failed 1-th time, retry later got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2026-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> single varchar col pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source2.yaml" got=2 expected=2 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> check validator can restart from previous position on fail over restart dm-worker1 check diff failed 1-th time, retry later wait process dm-master.test exit... wait process worker1 exit... wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 15107 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test without check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:08 CST 2024] <<<<<< test case ha_cases_2 success! >>>>>> start running case: [ha_cases2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:08 CST 2024] <<<<<< start test_pause_task >>>>>> [Sun May 5 20:56:08 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 restart dm-master wait process worker1 exit... process worker1 already exit wait process dm-worker.test exit... [Sun May 5 20:56:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" check diff successfully new_gtid1 94839261-0add-11ef-b944-ba9391cd6523:6 new_gtid2 94f14e08-0add-11ef-801d-ba9391cd6523:6 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master exit... [Sun May 5 20:56:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> [Sun May 5 20:56:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:10 CST 2024] <<<<<< test case lightning_load_task success! >>>>>> start running case: [lightning_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/run.sh... Verbose mode = false [Sun May 5 20:56:10 CST 2024] <<<<<< test case lightning_mode success! >>>>>> start running case: [metrics] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/run.sh... Verbose mode = false import prepare data start DM worker and master rpc addr 127.0.0.1:8262 is alive restart dm-worker2 [Sun May 5 20:56:10 CST 2024] <<<<<< finish DM-120 optimistic >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check dashboard data source check dashboard data source success [Sun May 5 20:56:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:56:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>> [Sun May 5 20:56:10 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>> [Sun May 5 20:56:10 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 wait process dm-master exit... process dm-master already exit Attempting encryption of all config, IAM users and policies on MinIO backend wait process dm-worker.test exit... wait process worker2 exit... [Sun May 5 20:56:11 CST 2024] <<<<<< start DM-121 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml" Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Sun May 5 20:56:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC98FD867AEC6B < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 12:56:12 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact dmctl test cmd: "query-status test" got=2 expected=2 wait process worker2 exit... process worker2 already exit [Sun May 5 20:56:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:56:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh: line 29: 27037 Terminated insert_data 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/metrics/source1.yaml" process dm-master.test already exit [Sun May 5 20:56:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:56:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:56:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Sun May 5 20:56:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> [Sun May 5 20:56:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/metrics/source2.yaml" check diff failed 2-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 restart dm-worker 1 wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 0-th time, will retry again rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/metrics/dm-task.yaml --remove-meta" [Sun May 5 20:56:16 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:16 CST 2024] <<<<<< test case async_checkpoint_flush success! >>>>>> start running case: [binlog_parse] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh... Verbose mode = false wait process dm-worker1 exit... wait process dm-worker.test exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" check diff failed 3-th time, retry later got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker1 exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=3 expected=3 wait for valid metric of dm_syncer_binlog_file{node="syncer",source_id="mysql-replica-01",task="test"} for 1-th time, got value: '1' wait for rpc addr 127.0.0.1:8261 alive the 2-th time process dm-worker1 already exit [Sun May 5 20:56:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8264 is alive [Sun May 5 20:56:17 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" rpc addr 127.0.0.1:18262 is alive [Sun May 5 20:56:18 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=3 expected=3 check diff failed 1-th time, retry later wait for valid metric of dm_syncer_binlog_file{node="master",source_id="mysql-replica-01",task="test"} for 1-th time, got value: '1' rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 4-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:56:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml " wait for valid metric of dm_syncer_replication_lag_sum{source_id="mysql-replica-01",task="test",worker="worker1"} for 1-th time, got value: '0' rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/binlog_parse/source1.yaml" prepare data start task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again --> check validator persist fail in the middle 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 check ddl lag done! check diff failed 5-th time, retry later got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase wait process dm-master.test exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... check full phase check diff successfully dmctl test cmd: "validation start test" prepare incremental data check incremental phase check diff successfully dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:22 CST 2024] <<<<<< test case gtid success! >>>>>> start running case: [ha_cases] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:22 CST 2024] <<<<<< start test_exclusive_relay >>>>>> start DM worker and master cluster [Sun May 5 20:56:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-standalone.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" got=2 expected=2 got=1 expected=1 got=1 expected=1 prepare incremental data 2 dmctl test cmd: "validation start test" check incremental phase 2 check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 got=1 expected=1 --> single datetime col pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:56:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:56:23 CST 2024] <<<<<< finish DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> [Sun May 5 20:56:23 CST 2024] <<<<<< start DM-INIT_SCHEMA optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff failed 6-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" [Sun May 5 20:56:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit check diff failed 7-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1 worker2" wait process dm-worker.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check task result run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" dmctl test cmd: "list-member --worker" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source show -s mysql-replica-02" got=1 expected=1 1 dm-master alive check diff failed 8-th time, retry later restart dm-master 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 1-th time, retry later use sync_diff_inspector to check increment data check diff successfully check diff successfully [Sun May 5 20:56:27 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master exit... got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:28 CST 2024] <<<<<< test case binlog_parse success! >>>>>> start running case: [case_sensitive] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml" run tidb sql failed 2-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml" wait process dm-master exit... process dm-master already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 start dumping SQLs into source pause tasks test dmctl test cmd: "pause-task test" dmctl test cmd: "pause-task test" check diff failed 2-th time, retry later dmctl test cmd: "query-status test" check diff failed 9-th time, retry later pause tasks test2 dmctl test cmd: "pause-task test2" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta" dmctl test cmd: "pause-task test2" wait process dm-worker.test exit... dmctl test cmd: "query-status test2" run tidb sql failed 3-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" [Sun May 5 20:56:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:56:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: ""unit": "Sync"" [Sun May 5 20:56:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status t-ร‹!s`t" wait process dm-worker.test exit... check dml/skip lag done! check diff failed 10-th time, retry later dmctl test cmd: "validation status test" resume tasks test dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "resume-task test" got=0 expected=2 command: validation status test "stage": "Stopped" count: 0 != expected: 2, failed the 0-th time, will retry again got=3 expected=3 dmctl test cmd: "query-status test" resume tasks test2 dmctl test cmd: "resume-task test2" run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task t-ร‹!s`t" got=3 expected=3 dmctl test cmd: "resume-task test2" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "query-status test2" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... check diff failed at last dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:34 CST 2024] <<<<<< finish DM-121 pessimistic >>>>>> [Sun May 5 20:56:34 CST 2024] <<<<<< start DM-121 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 2-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:56:34 CST 2024] <<<<<< finish test_exclusive_relay >>>>>> [Sun May 5 20:56:34 CST 2024] <<<<<< start test_exclusive_relay_2 >>>>>> start DM worker and master cluster [Sun May 5 20:56:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-standalone.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:56:34 CST 2024] <<<<<< finish DM-INIT_SCHEMA optimistic >>>>>> [Sun May 5 20:56:34 CST 2024] <<<<<< start DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 got=2 expected=2 got=2 expected=2 got=2 expected=2 got=2 expected=2 --> after resume and persist again, meta data should be consistent dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml " wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait pattern dm-worker1.toml exit... dmctl test cmd: "validation status test" check diff failed 1-th time, retry later wait pattern dm-worker1.toml exit... got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait pattern dm-worker1.toml exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:56:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker 2 dmctl test cmd: "validation status test" got=4 expected=4 got=2 expected=2 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later wait pattern dm-worker1.toml exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Sun May 5 20:56:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 3-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" [Sun May 5 20:56:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker2 exit... rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 wait pattern dm-worker2.toml exit... check diff failed 2-th time, retry later got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again wait pattern dm-worker2.toml exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:56:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check zero job done! wait pattern dm-worker1.toml exit... wait pattern dm-worker2.toml exit... --> check validate success after insert data manually dmctl test cmd: "validation status test" got=4 expected=4 got=2 expected=2 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again wait pattern dm-worker1.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... run tidb sql failed 4-th time, retry later wait pattern dm-worker1.toml exit... wait pattern dm-worker2.toml exit... dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Sun May 5 20:56:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Sun May 5 20:56:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later got=1 expected=1 got=1 expected=1 got=1 expected=1 --> compound pk (datetime, timestamp, int, varchar) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=4 expected=4 got=2 expected=2 got=2 expected=2 got=2 expected=2 got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8263 is alive 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... run tidb sql failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive make a dml job check diff successfully dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 restart dm-master got=2 expected=2 [Sun May 5 20:56:41 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit check diff failed 4-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 15779 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with special task-name and check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 wait process dm-master.test exit... wait process dm-master exit... rpc addr 127.0.0.1:8264 is alive kill dm-worker1 wait process dm-worker.test exit... You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker1 exit... wait process dm-worker.test exit... wait process dm-master exit... process dm-master already exit check diff failed 5-th time, retry later * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC9904C76D7EA6 < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 12:56:43 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --name worker3" got=1 expected=1 [Sun May 5 20:56:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:44 CST 2024] <<<<<< finish test_query_timeout >>>>>> [Sun May 5 20:56:44 CST 2024] <<<<<< start test_regexpr_router regexpr-task.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:56:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:45 CST 2024] <<<<<< test case metrics success! >>>>>> [Pipeline] } rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member --name worker3" Cache not saved (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm already exists) rpc addr 127.0.0.1:8261 is alive [Pipeline] // cache [Pipeline] } wait process dm-worker.test exit... [Pipeline] // dir [Pipeline] } got=1 expected=1 dmctl test cmd: "list-member --name worker1" got=1 expected=1 kill dm-worker2 [Pipeline] // withCredentials rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] } wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:56:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // timeout [Pipeline] } check diff failed 6-th time, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-worker.test exit... wait process dm-worker2 exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:56:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 restart dm-worker 2 [Sun May 5 20:56:48 CST 2024] <<<<<< test case validator_checkpoint success! >>>>>> start running case: [incompatible_ddl_changes] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 7-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "operate-source show -s mysql-replica-02" got=1 expected=1 [Sun May 5 20:56:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-worker2 exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker2" got=1 expected=1 1 dm-master alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time 3 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" check diff failed 8-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit [Sun May 5 20:56:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=3 expected=3 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:56:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/source1.yaml" dmctl test cmd: "validation status test" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-task.yaml " got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/4\/3" count: 0 != expected: 1, failed the 0-th time, will retry again wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... check diff failed 9-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:56:53 CST 2024] <<<<<< finish DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> [Sun May 5 20:56:53 CST 2024] <<<<<< start DM-RECOVER_LOCK optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... check diff failed 10-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" wait process dm-worker.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table without primary key 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... check diff failed at last dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" [Sun May 5 20:56:56 CST 2024] <<<<<< finish test_regexpr_router regexpr-task.yaml >>>>>> [Sun May 5 20:56:56 CST 2024] <<<<<< start test_regexpr_router regexpr-task-lightning.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:56:56 CST 2024] <<<<<< finish test_exclusive_relay_2 >>>>>> [Sun May 5 20:56:56 CST 2024] <<<<<< start test_last_bound >>>>>> [Sun May 5 20:56:56 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table [Sun May 5 20:56:56 CST 2024] <<<<<< finish DM-121 optimistic >>>>>> prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" [Sun May 5 20:56:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" [Sun May 5 20:56:57 CST 2024] <<<<<< start DM-122 pessimistic >>>>>> got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:56:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:56:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time import prepare data start DM worker and master cluster [Sun May 5 20:56:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master1.toml >>>>>> [Sun May 5 20:56:58 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master2.toml >>>>>> [Sun May 5 20:56:58 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" restart dm-master wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test event reorganize table partition count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" wait process dm-worker.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-master exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check task result run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master exit... process dm-master already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:57:01 CST 2024] <<<<<< finish DM-122 pessimistic >>>>>> [Sun May 5 20:57:01 CST 2024] <<<<<< start DM-122 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time run tidb sql failed 2-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sun May 5 20:57:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" [Sun May 5 20:57:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:57:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" [Sun May 5 20:57:03 CST 2024] <<<<<< finish DM-122 optimistic >>>>>> got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" [Sun May 5 20:57:04 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:57:04 CST 2024] <<<<<< start DM-123 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" run tidb sql failed 3-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit [Sun May 5 20:57:04 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "stop-task test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive restart dm-master wait process dm-worker.test exit... check diff successfully wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-task.yaml " wait pattern dm-worker1.toml exit... wait process dm-master exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait process dm-master.test exit... process dm-master.test already exit wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Sun May 5 20:57:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:07 CST 2024] <<<<<< finish test_regexpr_router regexpr-task-lightning.yaml >>>>>> [Sun May 5 20:57:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master exit... process dm-master already exit run tidb sql failed 2-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:57:08 CST 2024] <<<<<< finish DM-123 pessimistic >>>>>> [Sun May 5 20:57:08 CST 2024] <<<<<< start DM-123 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 wait pattern dm-worker2.toml exit... rpc addr 127.0.0.1:8261 is alive wait process dm-worker.test exit... [Sun May 5 20:57:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... dmctl test cmd: "query-status test" got=2 expected=2 wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time run tidb sql failed 3-th time, retry later wait pattern dm-worker2.toml exit... [Sun May 5 20:57:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table is deleted on downstream 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Sun May 5 20:57:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sun May 5 20:57:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time use sync_diff_inspector to check increment data check diff successfully [Sun May 5 20:57:11 CST 2024] <<<<<< finish test_running >>>>>> worker1bound "mysql-replica-01" worker2bound "mysql-replica-02" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-task1.yaml " check diff successfully dmctl test cmd: "stop-task test" [Sun May 5 20:57:12 CST 2024] <<<<<< finish DM-123 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 16453 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with special task-name and without check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8261 is alive check diff failed 1-th time, retry later wait process dm-worker.test exit... [Sun May 5 20:57:14 CST 2024] <<<<<< start DM-124 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task /tmp/dm_test/all_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 kill dm-worker1 check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC990C25F128A2 < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 12:57:15 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:57:15 CST 2024] <<<<<< finish DM-124 optimistic >>>>>> wait process dm-worker1 exit... check diff successfully dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" [Sun May 5 20:57:16 CST 2024] <<<<<< finish DM-RECOVER_LOCK optimistic >>>>>> run DM_DropAddColumn case #0 [Sun May 5 20:57:16 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "pause-task t-ร‹!s`t" dmctl test cmd: "operate-schema set -s mysql-replica-01 t-ร‹!s`t -d all_mode -t no_diff /tmp/dm_test/all_mode/schema.sql" dmctl test cmd: "resume-task t-ร‹!s`t" wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" [Sun May 5 20:57:16 CST 2024] <<<<<< start DM-125 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait pattern dm-worker1.toml exit... got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" wait pattern dm-worker1.toml exit... got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" [Sun May 5 20:57:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait pattern dm-worker1.toml exit... got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker2 exit... got=2 expected=2 check diff failed 1-th time, retry later wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Sun May 5 20:57:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker1 [Sun May 5 20:57:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 wait process dm-worker.test exit... dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 wait pattern dm-worker2.toml exit... got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait pattern dm-worker2.toml exit... dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait pattern dm-worker2.toml exit... rpc addr 127.0.0.1:8262 is alive start worker2 [Sun May 5 20:57:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... wait pattern dm-worker2.toml exit... got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... [Sun May 5 20:57:20 CST 2024] <<<<<< finish DM-125 optimistic >>>>>> got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test event reorganize table partition count: 0 != expected: 1, failed the 0-th time, will retry again wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Sun May 5 20:57:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker1 --name worker2" got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... [Sun May 5 20:57:21 CST 2024] <<<<<< start DM-126 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" kill dm-worker1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table in schema-tracker has less column than binlog 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive check diff failed 2-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit [Sun May 5 20:57:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" check diff successfully got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" check diff failed 3-th time, retry later wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-worker2 exit... got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:57:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Sun May 5 20:57:26 CST 2024] <<<<<< finish DM-126 optimistic >>>>>> wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker2 [Sun May 5 20:57:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... check diff failed at last dmctl test cmd: "binlog skip test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" [Sun May 5 20:57:27 CST 2024] <<<<<< start DM-127 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" [Sun May 5 20:57:27 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #1 [Sun May 5 20:57:27 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive start worker1 [Sun May 5 20:57:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sun May 5 20:57:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time prepare source data wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member --name worker2 --name worker1" got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 check diff successfully restart dm-master kill dm-worker1 wait process dm-worker.test exit... wait process dm-master exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sun May 5 20:57:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker1 exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process dm-master exit... process dm-master already exit check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Sun May 5 20:57:31 CST 2024] <<<<<< finish DM-127 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:31 CST 2024] <<<<<< test case incompatible_ddl_changes success! >>>>>> [Pipeline] } wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:31 CST 2024] <<<<<< test case case_sensitive success! >>>>>> Cache not saved (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1907/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } wait process dm-worker2 exit... [Pipeline] // withCredentials start running case: [checkpoint_transaction] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Sun May 5 20:57:32 CST 2024] <<<<<< start DM-128 pessimistic >>>>>> [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } after restart dm-worker, task should resume automatically dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml" [Pipeline] // stage [Pipeline] } rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-worker1.toml >>>>>> [Sun May 5 20:57:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker3 [Sun May 5 20:57:33 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/source1.yaml" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-task.yaml " got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> pk column of downstream table not in range of binlog column 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8264 is alive start worker4 [Sun May 5 20:57:34 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time HTTP 127.0.0.1:8261/apis/v1alpha1/status/t-ร‹!s`t is alive wait process dm-master.test exit... rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "list-member --name worker3 --name worker4" got=1 expected=1 got=1 expected=1 check diff successfully rpc addr 127.0.0.1:8261 is alive check diff successfully dmctl test cmd: "stop-task test" check log contain failed 1-th time, retry later dmctl test cmd: "start-relay -s mysql-replica-01 worker3" [Sun May 5 20:57:36 CST 2024] <<<<<< finish DM-128 pessimistic >>>>>> [Sun May 5 20:57:36 CST 2024] <<<<<< start DM-128 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker4" wait process dm-worker.test exit... kill dm-master1 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "shard-ddl-lock" check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" [Sun May 5 20:57:38 CST 2024] <<<<<< finish DM-128 optimistic >>>>>> got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker.test exit... error check wait process dm-master.test exit... got=4 expected=4 check diff successfully kill dm-worker3 wait process tidb-server exit... process tidb-server already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:57:39 CST 2024] <<<<<< start DM-129 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 17145 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test error check success Starting TiDB on port 4000 Verifying TiDB is started... [2024/05/05 20:57:39.845 +08:00] [ERROR] [tempDir.go:76] ["The current temporary storage dir has been occupied by another instance, check tmp-storage-path config and make sure they are different."] [TempStoragePath="/tmp/1000_tidb/MC4wLjAuMDo0MDAwLzAuMC4wLjA6MTAwODA=/tmp-storage"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/pkg/util/disk.InitializeTempDir\n\t/home/jenkins/agent/workspace/build_tidb_multi_branch_master/go/src/github.com/pingcap/tidb/pkg/util/disk/tempDir.go:76\nmain.main\n\t/home/jenkins/agent/workspace/build_tidb_multi_branch_master/go/src/github.com/pingcap/tidb/cmd/tidb-server/main.go:287\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"] [2024/05/05 20:57:39.845 +08:00] [FATAL] [terror.go:309] ["unexpected error"] [error="fslock: lock is held"] [stack="github.com/pingcap/tidb/pkg/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/build_tidb_multi_branch_master/go/src/github.com/pingcap/tidb/pkg/parser/terror/terror.go:309\nmain.main\n\t/home/jenkins/agent/workspace/build_tidb_multi_branch_master/go/src/github.com/pingcap/tidb/cmd/tidb-server/main.go:288\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"] [stack="github.com/pingcap/tidb/pkg/parser/terror.MustNil\n\t/home/jenkins/agent/workspace/build_tidb_multi_branch_master/go/src/github.com/pingcap/tidb/pkg/parser/terror/terror.go:309\nmain.main\n\t/home/jenkins/agent/workspace/build_tidb_multi_branch_master/go/src/github.com/pingcap/tidb/cmd/tidb-server/main.go:288\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"] wait process dm-worker3 exit... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 2-th time, retry later wait process dm-worker3 exit... process dm-worker3 already exit kill dm-worker4 Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" start check_worker_ungraceful_stop_with_retry times: 0 [Sun May 5 20:57:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker4 exit... start check_worker_ungraceful_stop_with_retry times: 1 check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" wait process dm-worker4 exit... process dm-worker4 already exit dmctl test cmd: "list-member --name worker3 --name worker4" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' got=2 expected=2 start worker1 [Sun May 5 20:57:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" start check_worker_ungraceful_stop_with_retry times: 2 check_worker_ungraceful_stop_with_retry success after retry: 2 after ungraceful exit data in source count: 31 data in tidb count: 17 ungraceful stop test success [Sun May 5 20:57:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sun May 5 20:57:43 CST 2024] <<<<<< finish DM-129 pessimistic >>>>>> [Sun May 5 20:57:43 CST 2024] <<<<<< start DM-129 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive start worker2 [Sun May 5 20:57:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "resume-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker1 --name worker2" got=1 expected=1 got=1 expected=1 [Sun May 5 20:57:45 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #2 [Sun May 5 20:57:45 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive num1 1 num2 2 [Sun May 5 20:57:45 CST 2024] <<<<<< finish test_last_bound >>>>>> [Sun May 5 20:57:45 CST 2024] <<<<<< start test_config_name >>>>>> [Sun May 5 20:57:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join1.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:45 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/ha_cases/dm-master-join2.toml >>>>>> [Sun May 5 20:57:45 CST 2024] <<<<<< finish DM-129 optimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> stopped validator fail over 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "query-status test" got=2 expected=2 [Sun May 5 20:57:46 CST 2024] <<<<<< start DM-130 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully restart dm-master check diff failed 2-th time, retry later wait process dm-master.test exit... check log contain failed 1-th time (file not exist), retry later [Sun May 5 20:57:47 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/ha_cases/dm-master-join2.toml >>>>>> wait process dm-master exit... rpc addr 127.0.0.1:8361 is alive [Sun May 5 20:57:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8262 is alive [Sun May 5 20:57:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/ha_cases/dm-worker2.toml >>>>>> wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun May 5 20:57:48 CST 2024] <<<<<< finish DM-130 pessimistic >>>>>> [Sun May 5 20:57:48 CST 2024] <<<<<< start DM-130 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "pause-relay -s mysql-replica-01" wait process dm-worker.test exit... wait process dm-master exit... process dm-master already exit dmctl test cmd: "resume-relay -s mysql-replica-01" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later [Sun May 5 20:57:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/ha_cases/dm-worker2.toml >>>>>> rpc addr 127.0.0.1:8263 is alive [Sun May 5 20:57:50 CST 2024] <<<<<< finish test_config_name >>>>>> [Sun May 5 20:57:50 CST 2024] <<<<<< start test_join_masters_and_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... [Sun May 5 20:57:51 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time relay logs dm-it-ff167e95-3f56-46b0-80c4-8ad3a45a27c4-dzwdc-95c25-bin.000002 relay.meta check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/all_mode/worker2/dumped_data.t-ร‹!s`t: No such file or directory worker2 auto removed dump files check no password in log dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" wait process dm-master.test exit... matched matched [Sun May 5 20:57:52 CST 2024] <<<<<< start test_source_and_target_with_empty_gtid >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb1 --action skip" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb2 --action skip" dmctl test cmd: "query-status test" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Sun May 5 20:57:53 CST 2024] <<<<<< finish DM-130 optimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... check diff failed 3-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun May 5 20:57:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... check diff failed 4-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... check diff failed 5-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "validation status test" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:57:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/all_mode/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:59 CST 2024] <<<<<< test case shardddl4 success! >>>>>> start running case: [shardddl4_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sun May 5 20:57:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" check diff failed 6-th time, retry later got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 gtid is empty start task and check stage dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task-no-gtid.yaml --remove-meta=true" wait process dm-master.test exit... check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Failed to start TiDB curl: (7) Failed connect to 127.0.0.1:8361; Connection refused curl: (7) Failed connect to 127.0.0.1:8461; Connection refused curl: (7) Failed connect to 127.0.0.1:8561; Connection refused curl: (7) Failed connect to 127.0.0.1:8661; Connection refused curl: (7) Failed connect to 127.0.0.1:8761; Connection refused curl: (7) Failed connect to 127.0.0.1:8262; Connection refused curl: (7) Failed connect to 127.0.0.1:8263; Connection refused curl: (7) Failed connect to 127.0.0.1:8264; Connection refused curl: (7) Failed connect to 127.0.0.1:18262; Connection refused curl: (7) Failed connect to 127.0.0.1:18263; Connection refused make: *** [dm_integration_test_in_group] Error 1 dmctl test cmd: "resume-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Post stage [Pipeline] sh check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff successfully ERROR 1146 (42S02) at line 1: Table 'all_mode.t2' doesn't exist run tidb sql failed 1-th time, retry later wait process dm-master.test exit... got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again [Sun May 5 20:58:01 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #3 [Sun May 5 20:58:01 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" + ls /tmp/dm_test cov.ha.dmctl.1714913403.2968.out cov.ha.dmctl.1714913405.3089.out cov.ha.dmctl.1714913407.3131.out cov.ha.dmctl.1714913412.3289.out cov.ha.dmctl.1714913416.3500.out cov.ha.dmctl.1714913419.3657.out cov.ha.dmctl.1714913428.3705.out cov.ha.dmctl.1714913440.3956.out cov.ha.dmctl.1714913440.3985.out cov.ha.dmctl.1714913440.4024.out cov.ha.dmctl.1714913441.4058.out cov.ha.dmctl.1714913441.4092.out cov.ha.dmctl.1714913442.4222.out cov.ha.dmctl.1714913449.4346.out cov.ha.dmctl.1714913455.4484.out cov.ha.dmctl.1714913464.4592.out cov.ha.dmctl.1714913469.4626.out cov.ha.dmctl.1714913471.4661.out cov.ha.dmctl.1714913471.4693.out cov.ha.dmctl.1714913473.4738.out cov.ha.master.out cov.ha.worker.8262.1714913402.out cov.ha.worker.8263.1714913404.out cov.ha.worker.8263.1714913430.out cov.ha.worker.8264.1714913418.out cov.ha.worker.8264.1714913439.out cov.import_v10x.dmctl.1714913362.840.out cov.import_v10x.dmctl.1714913362.877.out cov.import_v10x.dmctl.1714913363.912.out cov.import_v10x.master.out cov.import_v10x.worker.8262.1714913360.out cov.import_v10x.worker.8263.1714913361.out cov.new_collation_off.dmctl.1714913491.5201.out cov.new_collation_off.dmctl.1714913491.5255.out cov.new_collation_off.master.out cov.new_collation_off.worker.8262.1714913490.out cov.only_dml.dmctl.1714913500.5669.out cov.only_dml.dmctl.1714913501.5787.out cov.only_dml.dmctl.1714913502.5846.out cov.only_dml.dmctl.1714913503.5963.out cov.only_dml.dmctl.1714913505.6036.out cov.only_dml.dmctl.1714913506.6103.out cov.only_dml.dmctl.1714913507.6168.out cov.only_dml.dmctl.1714913508.6234.out cov.only_dml.dmctl.1714913509.6302.out cov.only_dml.dmctl.1714913510.6368.out cov.only_dml.dmctl.1714913511.6438.out cov.only_dml.dmctl.1714913513.6504.out cov.only_dml.dmctl.1714913514.6572.out cov.only_dml.dmctl.1714913515.6648.out cov.only_dml.dmctl.1714913516.6723.out cov.only_dml.dmctl.1714913517.6796.out cov.only_dml.dmctl.1714913518.6866.out cov.only_dml.dmctl.1714913519.6934.out cov.only_dml.dmctl.1714913520.7004.out cov.only_dml.dmctl.1714913522.7071.out cov.only_dml.dmctl.1714913523.7138.out cov.only_dml.dmctl.1714913524.7205.out cov.only_dml.dmctl.1714913525.7273.out cov.only_dml.dmctl.1714913526.7341.out cov.only_dml.dmctl.1714913527.7422.out cov.only_dml.dmctl.1714913528.7481.out cov.only_dml.dmctl.1714913530.7556.out cov.only_dml.dmctl.1714913531.7624.out cov.only_dml.dmctl.1714913532.7689.out cov.only_dml.dmctl.1714913533.7759.out cov.only_dml.dmctl.1714913534.7828.out cov.only_dml.dmctl.1714913535.7897.out cov.only_dml.dmctl.1714913536.7966.out cov.only_dml.dmctl.1714913538.8034.out cov.only_dml.dmctl.1714913539.8105.out cov.only_dml.dmctl.1714913540.8172.out cov.only_dml.dmctl.1714913541.8255.out cov.only_dml.dmctl.1714913542.8322.out cov.only_dml.dmctl.1714913543.8392.out cov.only_dml.master.out cov.only_dml.worker.8262.1714913499.out cov.only_dml.worker.8263.1714913500.out cov.openapi.dmctl.1714913565.9046.out cov.openapi.dmctl.1714913565.9085.out cov.openapi.dmctl.1714913566.9127.out cov.openapi.dmctl.1714913567.9169.out cov.openapi.dmctl.1714913568.9238.out cov.openapi.dmctl.1714913569.9334.out cov.openapi.dmctl.1714913570.9383.out cov.openapi.dmctl.1714913572.9552.out cov.openapi.dmctl.1714913572.9585.out cov.openapi.dmctl.1714913572.9633.out cov.openapi.dmctl.1714913573.9667.out cov.openapi.dmctl.1714913574.9736.out cov.openapi.dmctl.1714913574.9785.out cov.openapi.dmctl.1714913577.9934.out cov.openapi.dmctl.1714913578.10009.out cov.openapi.dmctl.1714913578.10043.out cov.openapi.dmctl.1714913578.9971.out cov.openapi.dmctl.1714913584.10314.out cov.openapi.dmctl.1714913584.10372.out cov.openapi.dmctl.1714913590.10676.out cov.openapi.dmctl.1714913590.10714.out cov.openapi.dmctl.1714913592.10909.out cov.openapi.dmctl.1714913593.10979.out cov.openapi.dmctl.1714913594.11066.out cov.openapi.dmctl.1714913595.11135.out cov.openapi.dmctl.1714913596.11206.out cov.openapi.dmctl.1714913603.11356.out cov.openapi.dmctl.1714913604.11432.out cov.openapi.dmctl.1714913604.11477.out cov.openapi.dmctl.1714913607.11578.out cov.openapi.dmctl.1714913611.11631.out cov.openapi.dmctl.1714913612.11719.out cov.openapi.dmctl.1714913616.11771.out cov.openapi.dmctl.1714913617.11847.out cov.openapi.dmctl.1714913621.11906.out cov.openapi.dmctl.1714913630.12225.out cov.openapi.dmctl.1714913630.12272.out cov.openapi.dmctl.1714913639.12560.out cov.openapi.dmctl.1714913639.12613.out cov.openapi.master.out cov.openapi.worker.8262.1714913563.out cov.openapi.worker.8262.1714913581.out cov.openapi.worker.8262.1714913587.out cov.openapi.worker.8262.1714913624.out cov.openapi.worker.8262.1714913640.out cov.openapi.worker.8262.1714913663.out cov.openapi.worker.8262.1714913684.out cov.openapi.worker.8263.1714913564.out cov.openapi.worker.8263.1714913582.out cov.openapi.worker.8263.1714913588.out cov.openapi.worker.8263.1714913625.out cov.openapi.worker.8263.1714913641.out cov.openapi.worker.8263.1714913664.out cov.openapi.worker.8263.1714913685.out cov.s3_dumpling_lightning.dmctl.1714913722.14800.out cov.s3_dumpling_lightning.dmctl.1714913723.14838.out cov.s3_dumpling_lightning.dmctl.1714913724.14911.out cov.s3_dumpling_lightning.dmctl.1714913751.15483.out cov.s3_dumpling_lightning.dmctl.1714913752.15519.out cov.s3_dumpling_lightning.dmctl.1714913753.15582.out cov.s3_dumpling_lightning.dmctl.1714913783.16164.out cov.s3_dumpling_lightning.dmctl.1714913783.16201.out cov.s3_dumpling_lightning.dmctl.1714913784.16262.out cov.s3_dumpling_lightning.dmctl.1714913815.16864.out cov.s3_dumpling_lightning.dmctl.1714913815.16901.out cov.s3_dumpling_lightning.dmctl.1714913816.16961.out cov.s3_dumpling_lightning.dmctl.1714913846.17535.out cov.s3_dumpling_lightning.dmctl.1714913847.17571.out cov.s3_dumpling_lightning.dmctl.1714913848.17645.out cov.s3_dumpling_lightning.master.out cov.s3_dumpling_lightning.worker.8262.1714913719.out cov.s3_dumpling_lightning.worker.8262.1714913749.out cov.s3_dumpling_lightning.worker.8262.1714913780.out cov.s3_dumpling_lightning.worker.8262.1714913813.out cov.s3_dumpling_lightning.worker.8263.1714913720.out cov.s3_dumpling_lightning.worker.8263.1714913750.out cov.s3_dumpling_lightning.worker.8263.1714913781.out cov.s3_dumpling_lightning.worker.8263.1714913814.out cov.sharding2.dmctl.1714913372.1313.out cov.sharding2.dmctl.1714913374.1431.out cov.sharding2.dmctl.1714913375.1477.out cov.sharding2.dmctl.1714913376.1513.out cov.sharding2.dmctl.1714913377.1615.out cov.sharding2.dmctl.1714913377.1656.out cov.sharding2.dmctl.1714913391.2245.out cov.sharding2.dmctl.1714913392.2392.out cov.sharding2.dmctl.1714913393.2433.out cov.sharding2.dmctl.1714913393.2523.out cov.sharding2.master.out cov.sharding2.worker.8262.1714913371.out cov.sharding2.worker.8262.1714913385.out cov.sharding2.worker.8262.1714913389.out cov.sharding2.worker.8263.1714913373.out cov.sharding2.worker.8263.1714913386.out cov.sharding2.worker.8263.1714913390.out downstream goroutines ha import_v10x new_collation_off only_dml openapi s3_dumpling_lightning s3_dumpling_lightning.downstream sharding2 sql_res.ha.txt sql_res.import_v10x.txt sql_res.new_collation_off.txt sql_res.only_dml.txt sql_res.openapi.txt sql_res.s3_dumpling_lightning.txt sql_res.sharding2.txt tidb.toml ++ find /tmp/dm_test/ -type f -name '*.log' + tar -cvzf log-G09.tar.gz /tmp/dm_test/ha/dmctl.1714913440.log /tmp/dm_test/ha/dmctl.1714913412.log /tmp/dm_test/ha/dmctl.1714913403.log /tmp/dm_test/ha/master4/log/dm-master.log /tmp/dm_test/ha/master4/log/stdout.log /tmp/dm_test/ha/dmctl.1714913449.log /tmp/dm_test/ha/dmctl.1714913469.log /tmp/dm_test/ha/dmctl.1714913419.log /tmp/dm_test/ha/dmctl.1714913473.log /tmp/dm_test/ha/dmctl.1714913416.log /tmp/dm_test/ha/dmctl.1714913441.log /tmp/dm_test/ha/dmctl.1714913471.log /tmp/dm_test/ha/master1/log/dm-master.log /tmp/dm_test/ha/master1/log/stdout.log /tmp/dm_test/ha/master3/log/dm-master.log /tmp/dm_test/ha/master3/log/stdout.log /tmp/dm_test/ha/worker2/log/dm-worker.log /tmp/dm_test/ha/worker2/log/stdout.log /tmp/dm_test/ha/master5/log/dm-master.log /tmp/dm_test/ha/master5/log/stdout.log /tmp/dm_test/ha/dmctl.1714913442.log /tmp/dm_test/ha/worker1/log/dm-worker.log /tmp/dm_test/ha/worker1/log/stdout.log /tmp/dm_test/ha/worker3/log/dm-worker.log /tmp/dm_test/ha/worker3/log/stdout.log /tmp/dm_test/ha/dmctl.1714913405.log /tmp/dm_test/ha/sync_diff_stdout.log /tmp/dm_test/ha/dmctl.1714913464.log /tmp/dm_test/ha/master6/log/dm-master.log /tmp/dm_test/ha/master6/log/stdout.log /tmp/dm_test/ha/dmctl.1714913455.log /tmp/dm_test/ha/master2/log/dm-master.log /tmp/dm_test/ha/master2/log/stdout.log /tmp/dm_test/ha/dmctl.1714913407.log /tmp/dm_test/ha/dmctl.1714913428.log /tmp/dm_test/openapi/master1/log/dm-master.log /tmp/dm_test/openapi/master1/log/stdout.log /tmp/dm_test/openapi/worker2/log/dm-worker.log /tmp/dm_test/openapi/worker2/log/stdout.log /tmp/dm_test/openapi/worker1/log/dm-worker.log /tmp/dm_test/openapi/worker1/log/stdout.log /tmp/dm_test/openapi/master2/log/dm-master.log /tmp/dm_test/openapi/master2/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714913536.log /tmp/dm_test/only_dml/dmctl.1714913533.log /tmp/dm_test/only_dml/dmctl.1714913534.log /tmp/dm_test/only_dml/dmctl.1714913543.log /tmp/dm_test/only_dml/dmctl.1714913514.log /tmp/dm_test/only_dml/dmctl.1714913519.log /tmp/dm_test/only_dml/dmctl.1714913509.log /tmp/dm_test/only_dml/dmctl.1714913525.log /tmp/dm_test/only_dml/dmctl.1714913517.log /tmp/dm_test/only_dml/dmctl.1714913502.log /tmp/dm_test/only_dml/dmctl.1714913518.log /tmp/dm_test/only_dml/dmctl.1714913523.log /tmp/dm_test/only_dml/dmctl.1714913516.log /tmp/dm_test/only_dml/dmctl.1714913510.log /tmp/dm_test/only_dml/dmctl.1714913528.log /tmp/dm_test/only_dml/dmctl.1714913522.log /tmp/dm_test/only_dml/dmctl.1714913531.log /tmp/dm_test/only_dml/dmctl.1714913507.log /tmp/dm_test/only_dml/dmctl.1714913513.log /tmp/dm_test/only_dml/dmctl.1714913524.log /tmp/dm_test/only_dml/worker2/log/dm-worker.log /tmp/dm_test/only_dml/worker2/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714913515.log /tmp/dm_test/only_dml/dmctl.1714913501.log /tmp/dm_test/only_dml/dmctl.1714913532.log /tmp/dm_test/only_dml/dmctl.1714913539.log /tmp/dm_test/only_dml/dmctl.1714913503.log /tmp/dm_test/only_dml/dmctl.1714913505.log /tmp/dm_test/only_dml/dmctl.1714913530.log /tmp/dm_test/only_dml/worker1/log/dm-worker.log /tmp/dm_test/only_dml/worker1/log/stdout.log /tmp/dm_test/only_dml/master/log/dm-master.log /tmp/dm_test/only_dml/master/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714913542.log /tmp/dm_test/only_dml/dmctl.1714913538.log /tmp/dm_test/only_dml/dmctl.1714913541.log /tmp/dm_test/only_dml/dmctl.1714913540.log /tmp/dm_test/only_dml/sync_diff_stdout.log /tmp/dm_test/only_dml/dmctl.1714913526.log /tmp/dm_test/only_dml/dmctl.1714913520.log /tmp/dm_test/only_dml/dmctl.1714913527.log /tmp/dm_test/only_dml/dmctl.1714913508.log /tmp/dm_test/only_dml/dmctl.1714913535.log /tmp/dm_test/only_dml/dmctl.1714913506.log /tmp/dm_test/only_dml/dmctl.1714913511.log /tmp/dm_test/only_dml/dmctl.1714913500.log /tmp/dm_test/goroutines/stack/log/worker-18262.log /tmp/dm_test/goroutines/stack/log/master-8661.log /tmp/dm_test/goroutines/stack/log/worker-8264.log /tmp/dm_test/goroutines/stack/log/worker-8262.log /tmp/dm_test/goroutines/stack/log/worker-8263.log /tmp/dm_test/goroutines/stack/log/worker-18263.log /tmp/dm_test/goroutines/stack/log/master-8461.log /tmp/dm_test/goroutines/stack/log/master-8261.log /tmp/dm_test/goroutines/stack/log/master-8561.log /tmp/dm_test/goroutines/stack/log/master-8761.log /tmp/dm_test/goroutines/stack/log/master-8361.log /tmp/dm_test/s3_dumpling_lightning.downstream/tidb.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv/db/000005.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/region-meta/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/hot-region/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv.log /tmp/dm_test/downstream/tidb/log/tidb.log /tmp/dm_test/sharding2/dmctl.1714913376.log /tmp/dm_test/sharding2/dmctl.1714913377.log /tmp/dm_test/sharding2/dmctl.1714913374.log /tmp/dm_test/sharding2/dmctl.1714913372.log /tmp/dm_test/sharding2/dmctl.1714913375.log /tmp/dm_test/sharding2/dmctl.1714913393.log /tmp/dm_test/sharding2/worker2/log/dm-worker.log /tmp/dm_test/sharding2/worker2/log/stdout.log /tmp/dm_test/sharding2/dmctl.1714913391.log /tmp/dm_test/sharding2/worker1/log/dm-worker.log /tmp/dm_test/sharding2/worker1/log/stdout.log /tmp/dm_test/sharding2/master/log/dm-master.log /tmp/dm_test/sharding2/master/log/stdout.log /tmp/dm_test/sharding2/sync_diff_stdout.log /tmp/dm_test/sharding2/dmctl.1714913392.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714913847.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714913846.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714913848.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/master/log/dm-master.log /tmp/dm_test/s3_dumpling_lightning/master/log/stdout.log /tmp/dm_test/new_collation_off/worker1/log/dm-worker.log /tmp/dm_test/new_collation_off/worker1/log/stdout.log /tmp/dm_test/new_collation_off/master/log/dm-master.log /tmp/dm_test/new_collation_off/master/log/stdout.log /tmp/dm_test/new_collation_off/sync_diff_stdout.log /tmp/dm_test/new_collation_off/dmctl.1714913491.log /tmp/dm_test/import_v10x/dmctl.1714913363.log /tmp/dm_test/import_v10x/dmctl.1714913362.log /tmp/dm_test/import_v10x/worker2/log/dm-worker.log /tmp/dm_test/import_v10x/worker2/log/stdout.log /tmp/dm_test/import_v10x/worker1/log/dm-worker.log /tmp/dm_test/import_v10x/worker1/log/stdout.log /tmp/dm_test/import_v10x/master/log/dm-master.log /tmp/dm_test/import_v10x/master/log/stdout.log /tmp/dm_test/import_v10x/sync_diff_stdout.log tar: Removing leading `/' from member names /tmp/dm_test/ha/dmctl.1714913440.log /tmp/dm_test/ha/dmctl.1714913412.log /tmp/dm_test/ha/dmctl.1714913403.log /tmp/dm_test/ha/master4/log/dm-master.log /tmp/dm_test/ha/master4/log/stdout.log /tmp/dm_test/ha/dmctl.1714913449.log /tmp/dm_test/ha/dmctl.1714913469.log /tmp/dm_test/ha/dmctl.1714913419.log /tmp/dm_test/ha/dmctl.1714913473.log /tmp/dm_test/ha/dmctl.1714913416.log /tmp/dm_test/ha/dmctl.1714913441.log /tmp/dm_test/ha/dmctl.1714913471.log /tmp/dm_test/ha/master1/log/dm-master.log /tmp/dm_test/ha/master1/log/stdout.log /tmp/dm_test/ha/master3/log/dm-master.log /tmp/dm_test/ha/master3/log/stdout.log /tmp/dm_test/ha/worker2/log/dm-worker.log /tmp/dm_test/ha/worker2/log/stdout.log /tmp/dm_test/ha/master5/log/dm-master.log /tmp/dm_test/ha/master5/log/stdout.log /tmp/dm_test/ha/dmctl.1714913442.log /tmp/dm_test/ha/worker1/log/dm-worker.log /tmp/dm_test/ha/worker1/log/stdout.log /tmp/dm_test/ha/worker3/log/dm-worker.log /tmp/dm_test/ha/worker3/log/stdout.log /tmp/dm_test/ha/dmctl.1714913405.log /tmp/dm_test/ha/sync_diff_stdout.log /tmp/dm_test/ha/dmctl.1714913464.log /tmp/dm_test/ha/master6/log/dm-master.log /tmp/dm_test/ha/master6/log/stdout.log /tmp/dm_test/ha/dmctl.1714913455.log /tmp/dm_test/ha/master2/log/dm-master.log /tmp/dm_test/ha/master2/log/stdout.log /tmp/dm_test/ha/dmctl.1714913407.log /tmp/dm_test/ha/dmctl.1714913428.log /tmp/dm_test/openapi/master1/log/dm-master.log /tmp/dm_test/openapi/master1/log/stdout.log /tmp/dm_test/openapi/worker2/log/dm-worker.log /tmp/dm_test/openapi/worker2/log/stdout.log /tmp/dm_test/openapi/worker1/log/dm-worker.log /tmp/dm_test/openapi/worker1/log/stdout.log /tmp/dm_test/openapi/master2/log/dm-master.log /tmp/dm_test/openapi/master2/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714913536.log /tmp/dm_test/only_dml/dmctl.1714913533.log /tmp/dm_test/only_dml/dmctl.1714913534.log /tmp/dm_test/only_dml/dmctl.1714913543.log /tmp/dm_test/only_dml/dmctl.1714913514.log /tmp/dm_test/only_dml/dmctl.1714913519.log /tmp/dm_test/only_dml/dmctl.1714913509.log /tmp/dm_test/only_dml/dmctl.1714913525.log /tmp/dm_test/only_dml/dmctl.1714913517.log /tmp/dm_test/only_dml/dmctl.1714913502.log /tmp/dm_test/only_dml/dmctl.1714913518.log /tmp/dm_test/only_dml/dmctl.1714913523.log /tmp/dm_test/only_dml/dmctl.1714913516.log /tmp/dm_test/only_dml/dmctl.1714913510.log /tmp/dm_test/only_dml/dmctl.1714913528.log /tmp/dm_test/only_dml/dmctl.1714913522.log /tmp/dm_test/only_dml/dmctl.1714913531.log /tmp/dm_test/only_dml/dmctl.1714913507.log /tmp/dm_test/only_dml/dmctl.1714913513.log /tmp/dm_test/only_dml/dmctl.1714913524.log /tmp/dm_test/only_dml/worker2/log/dm-worker.log /tmp/dm_test/only_dml/worker2/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714913515.log /tmp/dm_test/only_dml/dmctl.1714913501.log /tmp/dm_test/only_dml/dmctl.1714913532.log /tmp/dm_test/only_dml/dmctl.1714913539.log /tmp/dm_test/only_dml/dmctl.1714913503.log /tmp/dm_test/only_dml/dmctl.1714913505.log /tmp/dm_test/only_dml/dmctl.1714913530.log /tmp/dm_test/only_dml/worker1/log/dm-worker.log /tmp/dm_test/only_dml/worker1/log/stdout.log /tmp/dm_test/only_dml/master/log/dm-master.log /tmp/dm_test/only_dml/master/log/stdout.log /tmp/dm_test/only_dml/dmctl.1714913542.log /tmp/dm_test/only_dml/dmctl.1714913538.log /tmp/dm_test/only_dml/dmctl.1714913541.log /tmp/dm_test/only_dml/dmctl.1714913540.log /tmp/dm_test/only_dml/sync_diff_stdout.log /tmp/dm_test/only_dml/dmctl.1714913526.log /tmp/dm_test/only_dml/dmctl.1714913520.log /tmp/dm_test/only_dml/dmctl.1714913527.log /tmp/dm_test/only_dml/dmctl.1714913508.log /tmp/dm_test/only_dml/dmctl.1714913535.log /tmp/dm_test/only_dml/dmctl.1714913506.log /tmp/dm_test/only_dml/dmctl.1714913511.log /tmp/dm_test/only_dml/dmctl.1714913500.log /tmp/dm_test/goroutines/stack/log/worker-18262.log /tmp/dm_test/goroutines/stack/log/master-8661.log /tmp/dm_test/goroutines/stack/log/worker-8264.log /tmp/dm_test/goroutines/stack/log/worker-8262.log /tmp/dm_test/goroutines/stack/log/worker-8263.log /tmp/dm_test/goroutines/stack/log/worker-18263.log /tmp/dm_test/goroutines/stack/log/master-8461.log /tmp/dm_test/goroutines/stack/log/master-8261.log /tmp/dm_test/goroutines/stack/log/master-8561.log /tmp/dm_test/goroutines/stack/log/master-8761.log /tmp/dm_test/goroutines/stack/log/master-8361.log /tmp/dm_test/s3_dumpling_lightning.downstream/tidb.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv/db/000005.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/region-meta/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/pd/hot-region/000001.log /tmp/dm_test/s3_dumpling_lightning.downstream/tikv.log /tmp/dm_test/downstream/tidb/log/tidb.log /tmp/dm_test/sharding2/dmctl.1714913376.log /tmp/dm_test/sharding2/dmctl.1714913377.log /tmp/dm_test/sharding2/dmctl.1714913374.log /tmp/dm_test/sharding2/dmctl.1714913372.log /tmp/dm_test/sharding2/dmctl.1714913375.log /tmp/dm_test/sharding2/dmctl.1714913393.log /tmp/dm_test/sharding2/worker2/log/dm-worker.log /tmp/dm_test/sharding2/worker2/log/stdout.log /tmp/dm_test/sharding2/dmctl.1714913391.log /tmp/dm_test/sharding2/worker1/log/dm-worker.log check diff failed 7-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sun May 5 20:58:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time /tmp/dm_test/sharding2/worker1/log/stdout.log /tmp/dm_test/sharding2/master/log/dm-master.log /tmp/dm_test/sharding2/master/log/stdout.log /tmp/dm_test/sharding2/sync_diff_stdout.log /tmp/dm_test/sharding2/dmctl.1714913392.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714913847.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714913846.log /tmp/dm_test/s3_dumpling_lightning/dmctl.1714913848.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker2/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/dm-worker.log /tmp/dm_test/s3_dumpling_lightning/worker1/log/stdout.log /tmp/dm_test/s3_dumpling_lightning/master/log/dm-master.log /tmp/dm_test/s3_dumpling_lightning/master/log/stdout.log /tmp/dm_test/new_collation_off/worker1/log/dm-worker.log /tmp/dm_test/new_collation_off/worker1/log/stdout.log /tmp/dm_test/new_collation_off/master/log/dm-master.log /tmp/dm_test/new_collation_off/master/log/stdout.log /tmp/dm_test/new_collation_off/sync_diff_stdout.log /tmp/dm_test/new_collation_off/dmctl.1714913491.log /tmp/dm_test/import_v10x/dmctl.1714913363.log /tmp/dm_test/import_v10x/dmctl.1714913362.log /tmp/dm_test/import_v10x/worker2/log/dm-worker.log /tmp/dm_test/import_v10x/worker2/log/stdout.log /tmp/dm_test/import_v10x/worker1/log/dm-worker.log /tmp/dm_test/import_v10x/worker1/log/stdout.log /tmp/dm_test/import_v10x/master/log/dm-master.log /tmp/dm_test/import_v10x/master/log/stdout.log /tmp/dm_test/import_v10x/sync_diff_stdout.log + ls -alh log-G09.tar.gz -rw-r--r-- 1 jenkins jenkins 1.5M May 5 20:58 log-G09.tar.gz [Pipeline] archiveArtifacts Archiving artifacts wait process dm-master.test exit... [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source1.yaml" [Pipeline] // withEnv dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Sun May 5 20:58:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] } check log contain failed 1-th time, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes check diff successfully pause task and check status dmctl test cmd: "pause-task test" kill finished with exit code 0 Sending interrupt signal to process Killing processes dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source2.yaml" make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 run tidb sql failed 1-th time, retry later restore config make: *** [dm_integration_test_in_group] Terminated /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh: line 53: 19374 Terminated sleep 2 restore time_zone ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G07' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' use sync_diff_inspector to check increment data check diff successfully check diff successfully [Sun May 5 20:57:56 CST 2024] <<<<<< finish test_pause_task >>>>>> [Sun May 5 20:57:56 CST 2024] <<<<<< start test_multi_task_reduce_and_restart_worker >>>>>> [Sun May 5 20:57:56 CST 2024] <<<<<< start test_multi_task_running >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G00' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 2 Finished: FAILURE