Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-7.5/pull_dm_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 Agent pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 is provisioned from template pingcap_tiflow_release-7_5_pull_dm_integration_test_307-v3g2m-hdgxg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c855283595e29145cc04d14cc66429840b83d24e" jenkins/label: "pingcap_tiflow_release-7_5_pull_dm_integration_test_307-v3g2m" name: "pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=a0833813-990e-4811-97cc-221e20139ca3 JENKINS_NODE_COOKIE=96034e01-6548-40c3-820b-45712c50dfc4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_dm_integration_test","buildid":"1787403523646820355","prowjobid":"a0833813-990e-4811-97cc-221e20139ca3","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","base_link":"https://github.com/pingcap/tiflow/commit/fd3a6e756d0d488c7e60bedb05e21e42320c3dc7","pulls":[{"number":10974,"author":"ti-chi-bot","sha":"93530e27720635216b5d5fcf531a48b801612005","title":"sink(ticdc): Add output-old-value config (#10915)","link":"https://github.com/pingcap/tiflow/pull/10974","commit_link":"https://github.com/pingcap/tiflow/pull/10974/commits/93530e27720635216b5d5fcf531a48b801612005","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-7.5/pull_dm_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#307 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787403523646820355 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_dm_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=635c5544ae8ad74f0d959461cc1d4bc8eba1b9bf RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-7_5_pull_dm_integration_test_307-v3g2m GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-24341cbb37398a3d76ef457a002bbb77ebf3752146c282247622af04dd7ac429 NODE_LABELS=pingcap_tiflow_release-7_5_pull_dm_integration_test_307-v3g2m pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/display/redirect BUILD_NUMBER=307 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3824969656=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 -c golang -- bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-7-5-pull-dm-integration-test-307-v-sscs5 -c golang -- bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 33655 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: c1ce109ab97ca786 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 1 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 08:46:45 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10974/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/api/v2/model.go, cdc/api/v2/model_test.go, docs/swagger/docs.go, docs/swagger/swagger.json, docs/swagger/swagger.yaml, pkg/cmd/util/helper_test.go, pkg/config/config_test_data.go, pkg/config/replica_config.go, pkg/config/replica_config_test.go, pkg/config/sink.go, pkg/orchestrator/reactor_state_test.go, pkg/sink/codec/common/config.go, pkg/sink/codec/open/open_protocol_encoder_test.go, pkg/sink/codec/open/open_protocol_message.go, tests/integration_tests/api_v2/cases.go, tests/integration_tests/api_v2/model.go] [Pipeline] echo diff file matched: pkg/cmd/util/helper_test.go [Pipeline] echo matched, some diff files full path start with dm/ or pkg/ or go.mod, run the dm integration test [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-fd3a6e7-93530e2) 202280448 bytes in 0.94 secs (214098781 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/.git/ .git HEAD is now at 93530e277 Merge remote-tracking branch 'upstream/release-7.5' into cherry-pick-10915-to-release-7.5 POST git-upload-pack (686 bytes) From https://github.com/pingcap/tiflow = [up to date] release-7.5 -> origin/release-7.5 = [up to date] refs/pull/10974/head -> origin/pr/10974/head Previous HEAD position was 93530e277 Merge remote-tracking branch 'upstream/release-7.5' into cherry-pick-10915-to-release-7.5 HEAD is now at fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) ๐Ÿšง Checkouting to base SHA:fd3a6e756d0d488c7e60bedb05e21e42320c3dc7... HEAD is now at fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: fd3a6e756d0d488c7e60bedb05e21e42320c3dc7 fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) 47417d775 mounter(ticdc): calculate row level checksum for timestmap by using UTC time zone (#10564) (#10646) 3ab47635f *: release-7.5 bump tidb dependencies (#10970) ๐Ÿšง Pre-merge heads of pull requests to base SHA: fd3a6e756d0d488c7e60bedb05e21e42320c3dc7 ... Updating fd3a6e756..93530e277 Fast-forward cdc/api/v2/model.go | 21 +++++++++- cdc/api/v2/model_test.go | 1 + docs/swagger/docs.go | 23 +++++++++++ docs/swagger/swagger.json | 23 +++++++++++ docs/swagger/swagger.yaml | 15 +++++++ pkg/cmd/util/helper_test.go | 2 + pkg/config/config_test_data.go | 15 +++++-- pkg/config/replica_config.go | 1 + pkg/config/replica_config_test.go | 3 ++ pkg/config/sink.go | 8 ++++ pkg/orchestrator/reactor_state_test.go | 3 ++ pkg/sink/codec/common/config.go | 8 ++++ pkg/sink/codec/open/open_protocol_encoder_test.go | 49 +++++++++++++++++++++++ pkg/sink/codec/open/open_protocol_message.go | 7 +++- tests/integration_tests/api_v2/cases.go | 2 + tests/integration_tests/api_v2/model.go | 32 +++++++++------ 16 files changed, 194 insertions(+), 19 deletions(-) ๐Ÿงพ Pre-merged result: 93530e27720635216b5d5fcf531a48b801612005 93530e277 Merge remote-tracking branch 'upstream/release-7.5' into cherry-pick-10915-to-release-7.5 fd3a6e756 * (ticdc): manually cherry-pick simple protocol to release-7.5 (#10898) 47417d775 mounter(ticdc): calculate row level checksum for timestmap by using UTC time zone (#10564) (#10646) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-fd3a6e7-93530e2 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./dm/tests/download-integration-test-binaries.sh release-7.5 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 876 0 --:--:-- --:--:-- --:--:-- 891 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1097 0 --:--:-- --:--:-- --:--:-- 1108 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 838 0 --:--:-- --:--:-- --:--:-- 854 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1453 0 --:--:-- --:--:-- --:--:-- 1464 Download binaries... >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/066b97f741e6fbef0740e073940cea08d13b3f43/centos7/tidb-server.tar.gz 2024-05-06 16:47:14 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/066b97f741e6fbef0740e073940cea08d13b3f43/centos7/tidb-server.tar.gz [511658098/511658098] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/78f4254e3f5adb48e3e1e2489065f5ccf6cf1815/centos7/pd-server.tar.gz 2024-05-06 16:47:33 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/78f4254e3f5adb48e3e1e2489065f5ccf6cf1815/centos7/pd-server.tar.gz [173223227/173223227] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/ee5bd74cfec316736bf6abc03f22955f88d53e24/centos7/tikv-server.tar.gz 2024-05-06 16:48:22 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/ee5bd74cfec316736bf6abc03f22955f88d53e24/centos7/tikv-server.tar.gz [810269604/810269604] -> "tmp/tikv-server.tar.gz" [1] >>> download tidb-tools.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/79f1fd1e50654cf89cee45eb63d9220a4d067188/centos7/tidb-tools.tar.gz 2024-05-06 16:48:39 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/79f1fd1e50654cf89cee45eb63d9220a4d067188/centos7/tidb-tools.tar.gz [190960542/190960542] -> "tmp/tidb-tools.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 16:48:43 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download gh-ost-binary-linux-20200828140552.tar.gz from https://github.com/github/gh-ost/releases/download/v1.1.0/gh-ost-binary-linux-20200828140552.tar.gz 2024-05-06 16:48:47 URL:https://objects.githubusercontent.com/github-production-release-asset-2e65be/54378638/ac8d1400-e94a-11ea-9664-8b516d434420?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240506%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240506T084844Z&X-Amz-Expires=300&X-Amz-Signature=cf0897f067fa73e5d70f4f5b7c11450227ad95e083086acf61748f65e19f4004&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=54378638&response-content-disposition=attachment%3B%20filename%3Dgh-ost-binary-linux-20200828140552.tar.gz&response-content-type=application%2Foctet-stream [5141587/5141587] -> "tmp/gh-ost-binary-linux-20200828140552.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.1G drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:48 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:48 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:48 bin -rwxr-xr-x. 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M Apr 30 10:51 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 42M Apr 30 10:50 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M Apr 30 10:51 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 30 10:50 pd-recover -rwxr-xr-x. 1 jenkins jenkins 102M Apr 30 10:50 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M Apr 30 10:51 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 30 10:51 regions-dump -rwxr-xr-x. 1 jenkins jenkins 29M Apr 30 10:51 stores-dump -rwxr-xr-x. 1 jenkins jenkins 131M May 6 11:41 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 6 16:05 tidb-server -rwxr-xr-x. 1 jenkins jenkins 353M May 1 23:57 tikv-server + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/gh-ost ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.1G drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:48 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:48 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:48 bin -rwxr-xr-x. 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M Apr 30 10:51 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 42M Apr 30 10:50 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M Apr 30 10:51 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 30 10:50 pd-recover -rwxr-xr-x. 1 jenkins jenkins 102M Apr 30 10:50 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M Apr 30 10:51 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M Apr 30 10:51 regions-dump -rwxr-xr-x. 1 jenkins jenkins 29M Apr 30 10:51 stores-dump -rwxr-xr-x. 1 jenkins jenkins 131M May 6 11:41 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 6 16:05 tidb-server -rwxr-xr-x. 1 jenkins jenkins 353M May 1 23:57 tikv-server + ./bin/tidb-server -V Release Version: v7.5.1-48-g066b97f741 Edition: Community Git Commit Hash: 066b97f741e6fbef0740e073940cea08d13b3f43 Git Branch: release-7.5 UTC Build Time: 2024-05-06 08:05:40 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 + ./bin/tikv-server -V TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: ee5bd74cfec316736bf6abc03f22955f88d53e24 Git Commit Branch: release-7.5 UTC Build Time: 2024-05-01 15:16:10 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (binary/pingcap/tiflow/dm-integration-test/rev-fd3a6e7-93530e2) 1487540736 bytes in 3.04 secs (489040099 bytes/sec) [Pipeline] { [Pipeline] sh + [[ ! -f bin/dm-master.test ]] + [[ ! -f bin/dm-test-tools/check_master_online ]] + [[ ! -f bin/dm-test-tools/check_worker_online ]] + echo 'Binaries already exist, skipping build...' Binaries already exist, skipping build... + ls -alh ./bin total 1.1G drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:48 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:48 .. -rwxr-xr-x. 1 jenkins jenkins 295M May 6 11:04 dm-master.test -rwxr-xr-x. 1 jenkins jenkins 282M May 6 11:07 dm-syncer.test drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:48 dm-test-tools -rwxr-xr-x. 1 jenkins jenkins 292M May 6 11:04 dm-worker.test -rwxr-xr-x. 1 jenkins jenkins 213M May 6 11:07 dmctl.test + ls -alh ./bin/dm-test-tools total 338M drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:48 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:48 .. -rwxr-xr-x. 1 jenkins jenkins 202M May 6 11:07 check_exit_safe_binlog -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_master_http_apis -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_master_online -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_master_online_http -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_worker_online + which ./bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test + which ./bin/dm-syncer.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test + which ./bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test + which ./bin/dmctl.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dmctl.test + which ./bin/dm-test-tools/check_master_online /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_master_online + which ./bin/dm-test-tools/check_worker_online /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_worker_online [Pipeline] } Cache not saved (binary/pingcap/tiflow/dm-integration-test/rev-fd3a6e7-93530e2 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/gh-ost ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.1G drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:48 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:48 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:48 bin -rwxr-xr-x. 1 jenkins jenkins 295M May 6 11:04 dm-master.test -rwxr-xr-x. 1 jenkins jenkins 282M May 6 11:07 dm-syncer.test drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:48 dm-test-tools -rwxr-xr-x. 1 jenkins jenkins 292M May 6 11:04 dm-worker.test -rwxr-xr-x. 1 jenkins jenkins 213M May 6 11:07 dmctl.test -rwxr-xr-x. 1 jenkins jenkins 9.8M May 6 16:48 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M May 6 16:48 minio -rwxr-xr-x. 1 jenkins jenkins 33M May 6 16:48 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 42M May 6 16:48 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M May 6 16:48 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 6 16:48 pd-recover -rwxr-xr-x. 1 jenkins jenkins 102M May 6 16:48 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M May 6 16:48 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 6 16:48 regions-dump -rwxr-xr-x. 1 jenkins jenkins 29M May 6 16:48 stores-dump -rwxr-xr-x. 1 jenkins jenkins 131M May 6 16:48 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 6 16:48 tidb-server -rwxr-xr-x. 1 jenkins jenkins 353M May 6 16:48 tikv-server + ls -alh ./bin/dm-test-tools total 338M drwxr-sr-x. 2 jenkins jenkins 4.0K May 6 16:48 . drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:48 .. -rwxr-xr-x. 1 jenkins jenkins 202M May 6 11:07 check_exit_safe_binlog -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_master_http_apis -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_master_online -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_master_online_http -rwxr-xr-x. 1 jenkins jenkins 34M May 6 11:07 check_worker_online [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 80.55 secs (34742240 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973-21bzh-424tg [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0 Created Pod: kubernetes jenkins-tiflow/dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x-lcl1w Created Pod: kubernetes jenkins-tiflow/dm-it-d2f017df-e8b0-4880-b546-69339fbf587c-ptvv5-8g839 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-d1923d50-416b-4926-b434-a176137f93a7-t9f88-1lf12 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57-371rm-j6rnk [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7-d52g3-5fh99 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973-21bzh-424tg is provisioned from template dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973-21bzh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8ac0d740973eda5275a0d9db72c3560a71ebdf44" jenkins/label: "dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973" name: "dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973-21bzh-424tg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973-21bzh-424tg" - name: "JENKINS_NAME" value: "dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973-21bzh-424tg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-8ebb79b1-9cc5-41ff-807a-b77f2365d973-21bzh-424tg in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { Agent dm-it-d2f017df-e8b0-4880-b546-69339fbf587c-ptvv5-8g839 is provisioned from template dm-it-d2f017df-e8b0-4880-b546-69339fbf587c-ptvv5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "58c453526cc2cd78d8634507bc9294f304fccab0" jenkins/label: "dm-it-d2f017df-e8b0-4880-b546-69339fbf587c" name: "dm-it-d2f017df-e8b0-4880-b546-69339fbf587c-ptvv5-8g839" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-d2f017df-e8b0-4880-b546-69339fbf587c-ptvv5-8g839" - name: "JENKINS_NAME" value: "dm-it-d2f017df-e8b0-4880-b546-69339fbf587c-ptvv5-8g839" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-d2f017df-e8b0-4880-b546-69339fbf587c-ptvv5-8g839 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] checkout The recommended git tool is: git [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@fc4d44; decorates RemoteLauncher[hudson.remoting.Channel@4a198b89:JNLP4-connect connection from 10.233.107.237/10.233.107.237:38354] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Created Pod: kubernetes jenkins-tiflow/dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1f1b200; decorates RemoteLauncher[hudson.remoting.Channel@6b957ccb:JNLP4-connect connection from 10.233.84.23/10.233.84.23:47948] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57-371rm-j6rnk is provisioned from template dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57-371rm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d522dd6f988fd8ec4d65272ec595ec5facfff48f" jenkins/label: "dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57" name: "dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57-371rm-j6rnk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57-371rm-j6rnk" - name: "JENKINS_NAME" value: "dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57-371rm-j6rnk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Running on dm-it-bd56607e-50aa-49eb-b4e5-325a4f629b57-371rm-j6rnk in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Agent dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0 is provisioned from template dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ea2b8cd975289726a6419eba902599c2aaff3dbf" jenkins/label: "dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1" name: "dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0" - name: "JENKINS_NAME" value: "dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] checkout Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git The recommended git tool is: git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate Agent dm-it-d1923d50-416b-4926-b434-a176137f93a7-t9f88-1lf12 is provisioned from template dm-it-d1923d50-416b-4926-b434-a176137f93a7-t9f88 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a877e29d7dc02261e073c2b48fdce9b6d22d0a21" jenkins/label: "dm-it-d1923d50-416b-4926-b434-a176137f93a7" name: "dm-it-d1923d50-416b-4926-b434-a176137f93a7-t9f88-1lf12" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-d1923d50-416b-4926-b434-a176137f93a7-t9f88-1lf12" - name: "JENKINS_NAME" value: "dm-it-d1923d50-416b-4926-b434-a176137f93a7-t9f88-1lf12" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Created Pod: kubernetes jenkins-tiflow/dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9 [Pipeline] { Running on dm-it-d1923d50-416b-4926-b434-a176137f93a7-t9f88-1lf12 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] node [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5565f227; decorates RemoteLauncher[hudson.remoting.Channel@213d3d73:JNLP4-connect connection from 10.233.123.85/10.233.123.85:40820] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1dc9d7d6; decorates RemoteLauncher[hudson.remoting.Channel@59a2292c:JNLP4-connect connection from 10.233.90.159/10.233.90.159:55586] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { Agent dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7-d52g3-5fh99 is provisioned from template dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7-d52g3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f8c60c3ab1e4dbef2f432adee596d94d6664c244" jenkins/label: "dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7" name: "dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7-d52g3-5fh99" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7-d52g3-5fh99" - name: "JENKINS_NAME" value: "dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7-d52g3-5fh99" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. Running on dm-it-2b2f534c-52e4-4728-97e0-5e4a37d066c7-d52g3-5fh99 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] { Agent dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x-lcl1w is provisioned from template dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "de671173f33d23d0dc2cee1952c7bc37f0d32259" jenkins/label: "dm-it-78976767-a1b4-484f-b9e8-871e87382bf7" name: "dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x-lcl1w" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x-lcl1w" - name: "JENKINS_NAME" value: "dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x-lcl1w" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x-lcl1w in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e952a59; decorates RemoteLauncher[hudson.remoting.Channel@39137222:JNLP4-connect connection from 10.233.72.69/10.233.72.69:59598] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] checkout Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 [Pipeline] { The recommended git tool is: git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] checkout Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) The recommended git tool is: git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/dm-it-57796927-3906-4812-8f6c-7799905ec7da-29frj-9bhvk Created Pod: kubernetes jenkins-tiflow/dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@61b353ed; decorates RemoteLauncher[hudson.remoting.Channel@3202ea56:JNLP4-connect connection from 10.233.107.54/10.233.107.54:36290] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5921d1c8; decorates RemoteLauncher[hudson.remoting.Channel@72defec5:JNLP4-connect connection from 10.233.106.120/10.233.106.120:55762] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:50 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:50 .. -rw------- 1 root 1000 1.7K May 6 08:50 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:50 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:50 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:50 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:50 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:50 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:50 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:50 server-key.pem [Pipeline] } [Pipeline] // container Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { Avoid second fetch [Pipeline] cache Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:50 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:50 .. -rw------- 1 root 1000 1.7K May 6 08:50 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:50 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:50 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:50 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:50 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:50 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:50 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:50 server-key.pem Created Pod: kubernetes jenkins-tiflow/dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1 Created Pod: kubernetes jenkins-tiflow/dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c-x3fhw-zljjt Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz is provisioned from template dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0bdb73457728ab66cf5d9a524cc8cbd5bb3d7c34" jenkins/label: "dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3" name: "dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz" - name: "JENKINS_NAME" value: "dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk is provisioned from template dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d388abdafdda5efed77f8d9679127124d15c492e" jenkins/label: "dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796" name: "dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk" - name: "JENKINS_NAME" value: "dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Agent dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9 is provisioned from template dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8410bcaa380c6f8ea727cab67c1a6fb5607ae38e" jenkins/label: "dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7" name: "dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9" - name: "JENKINS_NAME" value: "dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Agent dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1 is provisioned from template dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2e866975439125b15dffa5d072eb048d1cbfc301" jenkins/label: "dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8" name: "dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1" - name: "JENKINS_NAME" value: "dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Agent dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c-x3fhw-zljjt is provisioned from template dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c-x3fhw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0e6bb027047913c7537ef61d2e894c9dff9cbf8f" jenkins/label: "dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c" name: "dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c-x3fhw-zljjt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c-x3fhw-zljjt" - name: "JENKINS_NAME" value: "dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c-x3fhw-zljjt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-1382b66b-883d-4bdc-94fd-ac378a6b150c-x3fhw-zljjt in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 6.62 secs (422486467 bytes/sec) [Pipeline] { [Pipeline] } [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:50 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:50 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:50 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:50 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:50 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:50 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:50 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:50 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:50 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:50 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:50 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:50 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:50 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:50 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:50 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:50 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:50 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:50 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:50 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:50 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] cache Agent dm-it-57796927-3906-4812-8f6c-7799905ec7da-29frj-9bhvk is provisioned from template dm-it-57796927-3906-4812-8f6c-7799905ec7da-29frj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/307/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c8fbd47c873d315ca764f6086e4d77ceb168c9f8" jenkins/label: "dm-it-57796927-3906-4812-8f6c-7799905ec7da" name: "dm-it-57796927-3906-4812-8f6c-7799905ec7da-29frj-9bhvk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-57796927-3906-4812-8f6c-7799905ec7da-29frj-9bhvk" - name: "JENKINS_NAME" value: "dm-it-57796927-3906-4812-8f6c-7799905ec7da-29frj-9bhvk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-57796927-3906-4812-8f6c-7799905ec7da-29frj-9bhvk in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 27.26 secs (102670446 bytes/sec) [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] sh The recommended git tool is: git [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:51 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:51 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:50 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:50 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:50 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:51 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:51 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:51 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:51 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:51 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:51 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:51 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:51 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@567de5ec; decorates RemoteLauncher[hudson.remoting.Channel@7ae8c33f:JNLP4-connect connection from 10.233.90.1/10.233.90.1:59248] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@32eb7b5f; decorates RemoteLauncher[hudson.remoting.Channel@129ee3:JNLP4-connect connection from 10.233.106.105/10.233.106.105:34070] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@59da2c20; decorates RemoteLauncher[hudson.remoting.Channel@221e4e7d:JNLP4-connect connection from 10.233.127.218/10.233.127.218:43820] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4e1fe18d; decorates RemoteLauncher[hudson.remoting.Channel@7bded1ab:JNLP4-connect connection from 10.233.84.190/10.233.84.190:58938] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@33c87089; decorates RemoteLauncher[hudson.remoting.Channel@5fbd3852:JNLP4-connect connection from 10.233.88.171/10.233.88.171:34292] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // timeout [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] } [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] checkout The recommended git tool is: git + '[' TLS_GROUP == G01 ']' + echo 'run G01 test' run G01 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] // timeout [Pipeline] sh > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch + make dm_integration_test_in_group GROUP=G01 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] { [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] { [Pipeline] { [Pipeline] { Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) + '[' TLS_GROUP == G00 ']' + echo 'run G00 test' run G00 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G00 [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2d1ac1ef; decorates RemoteLauncher[hudson.remoting.Channel@8a2227d:JNLP4-connect connection from 10.233.100.159/10.233.100.159:55022] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 6 08:51 . drwxr-sr-x. 17 1000 1000 4.0K May 6 08:51 .. -rw-------. 1 root 1000 1.7K May 6 08:51 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 6 08:51 ca.pem -rw-r--r--. 1 root 1000 1.1K May 6 08:51 client-cert.pem -rw-------. 1 root 1000 1.7K May 6 08:51 client-key.pem -rw-------. 1 root 1000 1.7K May 6 08:51 private_key.pem -rw-r--r--. 1 root 1000 451 May 6 08:51 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 6 08:51 server-cert.pem -rw-------. 1 root 1000 1.7K May 6 08:51 server-key.pem > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] sh Commit message: "fix(br): use failpoint tidb-server instead (#2951)" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G01 Run cases: ha_cases3 ha_cases3_1 ha_master ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G00 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 6 08:51 . drwxr-sr-x. 17 1000 1000 4.0K May 6 08:51 .. -rw-------. 1 root 1000 1.7K May 6 08:51 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 6 08:51 ca.pem -rw-r--r--. 1 root 1000 1.1K May 6 08:51 client-cert.pem -rw-------. 1 root 1000 1.7K May 6 08:51 client-key.pem -rw-------. 1 root 1000 1.7K May 6 08:51 private_key.pem -rw-r--r--. 1 root 1000 451 May 6 08:51 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 6 08:51 server-cert.pem -rw-------. 1 root 1000 1.7K May 6 08:51 server-key.pem [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Run cases: ha_cases_1 ha_cases_2 ha_cases2 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:51 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:51 .. -rw------- 1 root 1000 1.7K May 6 08:51 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:51 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:51 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:51 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:51 server-key.pem [Pipeline] } [Pipeline] } [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:51 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:51 .. -rw------- 1 root 1000 1.7K May 6 08:51 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:51 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:51 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:51 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:51 server-key.pem [Pipeline] withEnv [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] } [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] // container [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:51 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:51 .. -rw------- 1 root 1000 1.7K May 6 08:51 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:51 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:51 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:51 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:51 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:51 server-key.pem VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh... Verbose mode = false Commit message: "fix(br): use failpoint tidb-server instead (#2951)" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:51:18 CST 2024] <<<<<< start test_stop_task >>>>>> [Mon May 6 16:51:18 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:51:19 CST 2024] <<<<<< start test_kill_master >>>>>> [Mon May 6 16:51:19 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 import prepare data start DM worker and master [Mon May 6 16:51:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master1.toml >>>>>> [Mon May 6 16:51:20 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master2.toml >>>>>> [Mon May 6 16:51:20 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time import prepare data start DM worker and master cluster [Mon May 6 16:51:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Mon May 6 16:51:21 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Mon May 6 16:51:21 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Mon May 6 16:51:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 16:51:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Mon May 6 16:51:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source2.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" [Mon May 6 16:51:26 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time [Mon May 6 16:51:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8264 is alive [Mon May 6 16:51:27 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" rpc addr 127.0.0.1:18262 is alive [Mon May 6 16:51:29 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 9.69 secs (288669575 bytes/sec) start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " [Pipeline] { [Pipeline] cache rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml " dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" apply increment data before restart dm-worker to ensure entering increment phase got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data check diff successfully [Mon May 6 16:51:34 CST 2024] <<<<<< finish test_running >>>>>> kill dm-master1 wait process dm-master1 exit... wait process dm-master1 exit... process dm-master1 already exit waiting 5 seconds use sync_diff_inspector to check increment data check diff successfully check diff successfully [Mon May 6 16:51:37 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 6.69 secs (417995089 bytes/sec) [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh got=2 expected=2 start dumping SQLs into source stop tasks test dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "query-status test" [Pipeline] sh got=1 expected=1 stop tasks test2 dmctl test cmd: "stop-task test2" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:51 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:51 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 6 16:51 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 6 16:51 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 .github -rw-r--r--. 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 bin drwxr-sr-x. 17 jenkins jenkins 4.0K May 6 16:51 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 6 16:51 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 6 16:51 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 6 16:51 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 6 16:51 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 examples -rw-r--r--. 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r--. 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K May 6 16:51 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 6 16:51 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 6 16:51 tests -rw-r--r--. 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache got=3 expected=3 dmctl test cmd: "query-status test2" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:51 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 6 16:51 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 6 16:51 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 6 16:51 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 .github -rw-r--r--. 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r--. 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r--. 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 bin drwxr-sr-x. 17 jenkins jenkins 4.0K May 6 16:51 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 6 16:51 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 6 16:51 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 6 16:51 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 6 16:51 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 examples -rw-r--r--. 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r--. 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 6 16:51 metrics drwxr-sr-x. 45 jenkins jenkins 4.0K May 6 16:51 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 6 16:51 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 6 16:51 tests -rw-r--r--. 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 6 16:51 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break got=1 expected=1 check task is running HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive check master2,3 are running dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "list-member --worker" got=2 expected=2 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml" use sync_diff_inspector to check increment2 data now! check diff successfully [Mon May 6 16:51:44 CST 2024] <<<<<< finish test_kill_master >>>>>> [Mon May 6 16:51:44 CST 2024] <<<<<< start test_kill_and_isolate_worker >>>>>> [Mon May 6 16:51:44 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test2" got=4 expected=4 wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 6.06 secs (461819150 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] sh wait process dm-master.test exit... + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:51 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:51 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:51 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:51 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:51 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:51 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:51 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:51 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:51 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:51 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:51 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:51 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:51 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } wait process dm-master.test exit... [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master cluster [Mon May 6 16:51:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Mon May 6 16:51:55 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Mon May 6 16:51:55 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Mon May 6 16:51:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" [Mon May 6 16:51:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data check diff successfully [Mon May 6 16:52:06 CST 2024] <<<<<< finish test_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=2 expected=2 kill dm-worker2 wait process dm-worker2 exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 15.42 secs (181484442 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:52 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:52 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:51 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:51 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:51 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:52 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:52 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:52 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:52 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:52 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status test" got=1 expected=1 [Mon May 6 16:52:09 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8264 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Mon May 6 16:52:10 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time [Pipeline] } [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] sh [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] cache + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:52 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:52 .. -rw------- 1 root 1000 1.7K May 6 08:52 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:52 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:52 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 server-key.pem rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3 worker4" got=3 expected=3 restart dm-worker3 wait process dm-worker3 exit... wait process dm-worker3 exit... process dm-worker3 already exit wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Mon May 6 16:52:14 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8264 is alive isolate dm-worker4 kill dm-worker4 wait process dm-worker4 exit... wait process dm-worker4 exit... process dm-worker4 already exit [Mon May 6 16:52:17 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 isolate dm-worker3 kill dm-worker3 wait process dm-worker3 exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 4.57 secs (612409381 bytes/sec) [Pipeline] { [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] // timeout [Pipeline] sh wait process dm-worker3 exit... process dm-worker3 already exit [Mon May 6 16:52:19 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> dmctl test cmd: "query-status test" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:52 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:52 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:52 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:52 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:52 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:52 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:52 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:52 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break got=1 expected=1 got=1 expected=1 disable isolate dm-worker4 kill dm-worker4 [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh + '[' TLS_GROUP == G06 ']' + echo 'run G06 test' run G06 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh + make dm_integration_test_in_group GROUP=G06 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:52 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:52 .. -rw------- 1 root 1000 1.7K May 6 08:52 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:52 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:52 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 server-key.pem wait process dm-worker4 exit... + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:52 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:52 .. -rw------- 1 root 1000 1.7K May 6 08:52 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:52 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:52 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 server-key.pem [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G06 Run cases: relay_interrupt safe_mode sequence_safe_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout [Pipeline] sh [Pipeline] } wait process dm-worker4 exit... process dm-worker4 already exit [Mon May 6 16:52:22 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=3 command: query-status test "stage": "Running" count: 1 != expected: 3, failed the 0-th time, will retry again [Pipeline] // timeout [Pipeline] sh + '[' TLS_GROUP == G05 ']' + echo 'run G05 test' run G05 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G05 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:52 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:52 .. -rw------- 1 root 1000 1.7K May 6 08:52 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:52 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:52 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 server-key.pem [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G05 Run cases: load_interrupt many_tables online_ddl load_interrupt not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [relay_interrupt] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetGlobalVariableFailed=return("server_uuid,1152") [Pipeline] } + '[' TLS_GROUP == G07 ']' + echo 'run G07 test' run G07 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] // timeout [Pipeline] sh + make dm_integration_test_in_group GROUP=G07 [Pipeline] container [Pipeline] { /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] // container [Mon May 6 16:52:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time + '[' TLS_GROUP == G03 ']' + echo 'run G03 test' run G03 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G03 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } [Pipeline] sh got=3 expected=3 query-status from all dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "pause-task test" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] container [Pipeline] { [Pipeline] cache /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test restart worker4 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:52 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:52 .. -rw------- 1 root 1000 1.7K May 6 08:52 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:52 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:52 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G07 Run cases: shardddl1 shardddl1_1 shardddl2 shardddl2_1 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait for rpc addr 127.0.0.1:8261 alive the 2-th time /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G03 Run cases: dmctl_advance dmctl_basic dmctl_command ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker4 exit... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [many_tables] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:52:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker4 exit... process dm-worker4 already exit [Mon May 6 16:52:27 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/0/source1.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [shardddl1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:52:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "resume-task test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [dmctl_advance] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm [Mon May 6 16:52:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. waiting for asynchronous relay and subtask to be started rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:52:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock unlock" dmctl test cmd: "shard-ddl-lock unlock lock_id --force-remove=invalid-force-remove" dmctl test cmd: "handle-error" dmctl test cmd: "handle-error test-task --binlog-pos mysql-bin:shoud-bin-digital skip" dmctl test cmd: "handle-error test-task replace alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "handle-error test-task wrong_operation" dmctl test cmd: "binlog" dmctl test cmd: "binlog skip test-task --binlog-pos mysql-bin:should-be-digital" dmctl test cmd: "binlog replace test-task alter table tb add column a int; alter table tb2 b int;" use sync_diff_inspector to check increment2 data now! check diff successfully [Mon May 6 16:52:30 CST 2024] <<<<<< finish test_kill_and_isolate_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog inject test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog list test-task --binlog-pos mysql-bin:should-be-digital" start prepare_data rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source1.yaml" [Mon May 6 16:52:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog wrong_operation test-task" dmctl test cmd: "binlog-schema" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-master.test exit... dmctl test cmd: "binlog-schema test" dmctl test cmd: "binlog-schema list test" dmctl test cmd: "query-status" dmctl test cmd: "binlog-schema update test" dmctl test cmd: "binlog-schema update test test_db test_table --from-source" dmctl test cmd: "binlog-schema update test test_db test_table --from-source --from-target" reset go failpoints, and need restart dm-worker then resume task, task will recover success rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source2.yaml" dmctl test cmd: "binlog-schema update test test_db test_table schema.sql --from-source" dmctl test cmd: "binlog-schema delete test" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:52:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-worker exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:52:34 CST 2024] <<<<<< test case dmctl_advance success! >>>>>> start running case: [dmctl_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm dmctl test cmd: "any command" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive wait process dm-worker exit... process dm-worker already exit [Mon May 6 16:52:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Mon May 6 16:52:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:52:36 CST 2024] <<<<<< start DM-COMPACT >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "any command" [Mon May 6 16:52:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive check_task_wrong_arg dmctl test cmd: "check-task" dmctl test cmd: "check-task not_exists_config_file" wait process dm-master.test exit... pause_relay_wrong_arg dmctl test cmd: "pause-relay wrong_arg" dmctl test cmd: "pause-relay" resume_relay_wrong_arg dmctl test cmd: "resume-relay wrong_arg" dmctl test cmd: "resume-relay" pause_task_wrong_arg dmctl test cmd: "pause-task" dmctl test cmd: "pause-task -s a -s b" resume_task_wrong_arg dmctl test cmd: "resume-task" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:52:39 CST 2024] <<<<<< test case ha_cases_1 success! >>>>>> start running case: [ha_cases_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:52:39 CST 2024] <<<<<< start test_kill_master_in_sync >>>>>> [Mon May 6 16:52:39 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "resume-task -s a -s b" query_status_wrong_arg dmctl test cmd: "query-status wrong_args_count more_than_one" dmctl test cmd: "query-status -s source-x task-y" start_task_wrong_arg dmctl test cmd: "start-task" dmctl test cmd: "start-task not_exists_config_file" stop_task_wrong_arg dmctl test cmd: "stop-task" dmctl test cmd: "stop-task -s a -s b" show_ddl_locks_wrong_arg dmctl test cmd: "shard-ddl-lock a b" dmctl test cmd: "show-ddl-locks a b" purge_relay_wrong_arg dmctl test cmd: "purge-relay wrong_arg" dmctl test cmd: "purge-relay" dmctl test cmd: "purge-relay --filename bin-000001 -s mysql-replica-01 -s mysql-replica-02" operate_source_empty_arg dmctl test cmd: "operate-source" dmctl test cmd: "operate-source create not_exists_config_file" dmctl test cmd: "operate-source invalid /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" dmctl test cmd: "operate-source stop /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" transfer_source_empty_arg dmctl test cmd: "transfer-source" dmctl test cmd: "transfer-source source-id" dmctl test cmd: "transfer-source source-id worker-id another-worker" start_relay_empty_arg dmctl test cmd: "start-relay" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 14.63 secs (191309803 bytes/sec) [Pipeline] { [Pipeline] sh dmctl test cmd: "start-relay wrong_arg" stop_relay_empty_arg dmctl test cmd: "stop-relay" dmctl test cmd: "stop-relay wrong_arg" import prepare data start DM worker and master cluster [Mon May 6 16:52:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Mon May 6 16:52:41 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Mon May 6 16:52:41 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 6 08:52 . drwxr-sr-x 17 1000 1000 4.0K May 6 08:52 .. -rw------- 1 root 1000 1.7K May 6 08:52 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 ca.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 client-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 client-key.pem -rw------- 1 root 1000 1.7K May 6 08:52 private_key.pem -rw-r--r-- 1 root 1000 451 May 6 08:52 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 6 08:52 server-cert.pem -rw------- 1 root 1000 1.7K May 6 08:52 server-key.pem source-id: mysql-replica-01 server-id: 123456 enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' enable-relay: false from: host: 127.0.0.1 user: root port: 3306 filters: - schema-pattern: "dmctl" table-pattern: "t_1" sql-pattern: ["alter table .* add column aaa int"] action: Ignore - schema-pattern: "dmctl" table-pattern: "t_2" sql-pattern: ["alter table .* add column aaa int"] action: Ignore dmctl test cmd: "operate-source stop /tmp/dm_test/dmctl_basic/source1-wrong.yaml mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" [Pipeline] // timeout [Pipeline] sh [Mon May 6 16:52:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:52 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:52 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:52 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:52 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:52 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:52 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:52 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:52 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break check diff failed 1-th time, retry later start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully read binlog from relay log failed, and will use remote binlog rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Mon May 6 16:52:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache + '[' TLS_GROUP == G02 ']' + echo 'run G02 test' run G02 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G02 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt dmctl test cmd: "operate-source show" [Mon May 6 16:52:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 2-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G02 Run cases: handle_error handle_error_2 handle_error_3 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" dmctl test cmd: "operate-source show -s mysql-replica-01" ############################################################ test case for operate tasks bound to a source ############################################################ dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/foo-task.yaml" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:52:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:52:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. check diff failed 3-th time, retry later version() 5.7.26-log version() 8.0.21 start running case: [handle_error] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:52:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " dmctl test cmd: "pause-task -s xxx" dmctl test cmd: "resume-task -s xxx" dmctl test cmd: "stop-task -s xxx" dmctl test cmd: "pause-task -s mysql-replica-01" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:52:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 4-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source1.yaml" [Mon May 6 16:52:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" check diff failed 5-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source2.yaml" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" [Mon May 6 16:52:51 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" use sync_diff_inspector to check increment data check diff successfully [Mon May 6 16:52:52 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 6-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb2 add column c int default 100; alter table handle_error.tb2 add primary key (c);" dmctl test cmd: "query-status test" wait process dm-master1 exit... got=2 expected=2 dmctl test cmd: "binlog inject test -s mysql-replica-01 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique(c);" dmctl test cmd: "binlog inject test -s mysql-replica-02 alter table handle_error.tb2 drop primary key; alter table handle_error.tb2 add unique(c);" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master1 exit... process dm-master1 already exit wait and check task running check diff failed 7-th time, retry later dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 9.75 secs (286969417 bytes/sec) [Pipeline] { [Pipeline] cache HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "stop-task test" [Mon May 6 16:52:56 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR >>>>>> [Mon May 6 16:52:56 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:52:57 CST 2024] <<<<<< finish DM-COMPACT >>>>>> finish prepare_data [Mon May 6 16:52:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:52:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/many_tables/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/baz-task.yaml" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" wait process dm-worker exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task.yaml " wait process dm-worker.test exit... dmctl test cmd: "pause-task -s mysql-replica-01" wait process dm-worker exit... process dm-worker already exit [Mon May 6 16:52:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Mon May 6 16:52:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetSessionVariableFailed=return("sql_mode,1152") dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:53:00 CST 2024] <<<<<< start DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "pause-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" [Mon May 6 16:53:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:53:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" run tidb sql failed 1-th time, retry later dmctl test cmd: "resume-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/1/source1.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" run tidb sql failed 2-th time, retry later dmctl test cmd: "transfer-source mysql-replica-01 worker1" dmctl test cmd: "transfer-source mysql-replica-01 worker2" kill worker2 0 in Load unit "progress": "0.00 %", waiting for asynchronous relay and subtask to be started wait process worker2 exit... run tidb sql failed 3-th time, retry later wait process worker2 exit... process worker2 already exit dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" restart worker2 [Mon May 6 16:53:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status" reset go failpoints, and need restart dm-worker then resume task, task will recover success rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-worker.test exit... run tidb sql failed 4-th time, retry later dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-01 worker2" dmctl test cmd: "start-relay -s mysql-replica-01" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:53:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "stop-relay -s mysql-replica-01" pause_relay_success dmctl test cmd: "pause-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=2 expected=2 dmctl test cmd: "resume-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl_check_task dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" 0 in Load unit "progress": "0.00 %", dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" run tidb sql failed 5-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 100 -w 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 0" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" check_task_optimistic dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task4.yaml" check_task_only_warning dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" run tidb sql failed 6-th time, retry later dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" check task with empty unit config dmctl test cmd: "check-task /tmp/empty-dump.yaml" --- name: empty-unit-task task-mode: all target-database: host: "127.0.0.1" port: 4000 user: "root" password: "" mysql-instances: - source-id: "mysql-replica-01" mydumper-config-name: "global" loader-config-name: "global" syncer-config-name: "global" block-allow-list: "instance" mydumpers: global: threads: 4 loaders: global: syncers: global: worker-count: 32 block-allow-list: instance: do-dbs: ["dmctl"] do-tables: - db-name: "dmctl" tbl-name: "~^t_[\\d]+"dmctl test cmd: "check-task /tmp/empty-load.yaml" dmctl test cmd: "check-task /tmp/empty-sync.yaml" start task with empty unit config dmctl test cmd: "start-task /tmp/empty-cfg.yaml" run tidb sql failed 7-th time, retry later dmctl test cmd: "config task empty-unit-task" dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/empty-unit-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/dmctl_basic/dm-task-error-database-config.yaml" dmctl_start_task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" 0 in Load unit "progress": "27.24 %", dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" run tidb sql failed 8-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 18.57 secs (150731502 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh dmctl test cmd: "query-status test_incremental_no_source_meta" got=3 expected=3 dmctl test cmd: "stop-task test_incremental_no_source_meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml " [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:52 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:52 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:52 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:52 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:52 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:52 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:52 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:52 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:52 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:52 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:52 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:52 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:53 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:53 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:53 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:53 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:53 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:53 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] cache check diff successfully dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully read binlog from relay log failed, and will use remote binlog run tidb sql failed 9-th time, retry later dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" wait process dm-worker.test exit... wait process dmctl_basic/worker1 exit... run tidb sql failed 10-th time, retry later 0 in Load unit "progress": "54.10 %", wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:53:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dmctl_basic/worker1 exit... process dmctl_basic/worker1 already exit use sync_diff_inspector to check increment data check diff successfully check diff successfully [Mon May 6 16:53:09 CST 2024] <<<<<< finish test_stop_task >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:53:18 CST 2024] <<<<<< test case ha_cases3 success! >>>>>> start running case: [ha_cases3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:53:18 CST 2024] <<<<<< start test_isolate_master_and_worker >>>>>> [Mon May 6 16:53:18 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master [Mon May 6 16:53:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> [Mon May 6 16:53:20 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> [Mon May 6 16:53:20 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Mon May 6 16:53:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive wait process dmctl_basic/worker2 exit... dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Mon May 6 16:53:21 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> [Mon May 6 16:53:21 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" run tidb sql failed 11-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source1.yaml" wait process dmctl_basic/worker2 exit... process dmctl_basic/worker2 already exit dmctl test cmd: "query-status" got=2 expected=2 got=2 expected=2 [Mon May 6 16:53:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:53:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 12-th time, retry later rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:53:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive config dmctl test cmd: "config" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Mon May 6 16:53:25 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> [Mon May 6 16:53:25 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source2.yaml" dmctl test cmd: "config hihi haha" dmctl test cmd: "config task haha" dmctl test cmd: "config master haha" dmctl test cmd: "config worker haha" dmctl test cmd: "config source haha" 0 in Load unit "progress": "79.96 %", dmctl test cmd: "get-config haha" dmctl test cmd: "get-config haha hihi" run tidb sql failed 13-th time, retry later dmctl test cmd: "get-config master haha" dmctl test cmd: "get-config worker haha" [Mon May 6 16:53:26 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "get-config source haha" dmctl test cmd: "get-config task haha" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "get-task-config haha" dmctl test cmd: "get-task-config haha hihi" dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01,mysql-replica-02 alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 6.73 secs (415561153 bytes/sec) [Pipeline] { dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8264 is alive [Mon May 6 16:53:27 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time dmctl test cmd: "operate-source stop mysql-replica-01" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source1.yaml" rpc addr 127.0.0.1:18262 is alive [Mon May 6 16:53:28 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time run tidb sql failed 14-th time, retry later dmctl test cmd: "operate-source stop mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source2.yaml" [Pipeline] } [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task2.yaml " [Pipeline] sh dmctl test cmd: "query-status test" [Pipeline] } dmctl test cmd: "stop-task test" [Mon May 6 16:53:29 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> [Mon May 6 16:53:29 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:53 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:53 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:53 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:53 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:53 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:53 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:53 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:53 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } dmctl test cmd: "start-task /tmp/dm_test/dmctl_basic/get_task.yaml" run tidb sql failed 15-th time, retry later dmctl test cmd: "query-status test" wait process dm-master.test exit... got=4 expected=4 got=2 expected=2 dmctl test cmd: "query-status test2" 0 in Sync unit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" got=4 expected=4 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully dmctl test cmd: "query-status test" check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-master exit... wait process dm-worker.test exit... wait process dm-master exit... process dm-master already exit [Mon May 6 16:53:33 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] } [Pipeline] } check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:53:34 CST 2024] <<<<<< finish DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later wait process dm-worker exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:53:35 CST 2024] <<<<<< test case relay_interrupt success! >>>>>> start running case: [safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:53:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] cache dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive wait process dm-worker exit... process dm-worker already exit [Mon May 6 16:53:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Mon May 6 16:53:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:53:36 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> [Mon May 6 16:53:36 CST 2024] <<<<<< start DM-LIST_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process worker1 exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:53:37 CST 2024] <<<<<< start DM-MULTIPLE_ROWS >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:53:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog list test" wait process worker1 exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:53:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:14435" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog inject test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:14627 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique (c);" dmctl test cmd: "query-status test" check log contain failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog skip test" wait process worker1 exit... run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" use sync_diff_inspector to check increment data check diff successfully check diff successfully [Mon May 6 16:53:38 CST 2024] <<<<<< finish test_multi_task_running >>>>>> [Mon May 6 16:53:38 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time rpc addr 127.0.0.1:8561 is alive wait process worker1 exit... check log contain failed 2-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task.yaml --remove-meta" [Mon May 6 16:53:41 CST 2024] <<<<<< finish DM-LIST_ERROR >>>>>> [Mon May 6 16:53:41 CST 2024] <<<<<< start DM-SKIP_ERROR >>>>>> wait process worker1 exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" start incremental_data wait process worker1 exit... check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process worker1 exit... run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 5.34 secs (524035604 bytes/sec) [Pipeline] { got=2 expected=2 dmctl test cmd: "stop-task test" [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { wait process worker1 exit... [Mon May 6 16:53:44 CST 2024] <<<<<< finish DM-SKIP_ERROR >>>>>> [Mon May 6 16:53:44 CST 2024] <<<<<< start DM-REPLACE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" [Pipeline] // timeout wait process dm-master.test exit... [Pipeline] sh [Pipeline] } [Pipeline] // timeout [Pipeline] sh wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:53:44 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time + '[' TLS_GROUP == G11 ']' + echo 'run G11 test' run G11 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G11 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] cache wait process worker1 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column new_col text, add column c int; alter table handle_error.tb1 add unique(c);" + '[' TLS_GROUP == G04 ']' + echo 'run G04 test' run G04 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ run tidb sql failed 2-th time, retry later run tidb sql failed 1-th time, retry later + make dm_integration_test_in_group GROUP=G04 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt rpc addr 127.0.0.1:8661 is alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G11 Run cases: adjust_gtid async_checkpoint_flush binlog_parse case_sensitive checkpoint_transaction check_task dm_syncer downstream_diff_index downstream_more_column drop_column_with_index duplicate_event expression_filter extend_column fake_rotate_event foreign_key full_mode gbk gtid ha_cases http_proxies lightning_load_task lightning_mode metrics new_collation_off only_dml openapi s3_dumpling_lightning sequence_sharding_optimistic sequence_sharding_removemeta shardddl_optimistic slow_relay_writer sql_mode sync_collation tracker_ignored_ddl validator_basic validator_checkpoint incompatible_ddl_changes ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process worker1 exit... wait process dm-worker.test exit... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G04 Run cases: import_goroutine_leak incremental_mode initial_unit import_goroutine_leak not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker.test exit... run tidb sql failed 3-th time, retry later dmctl test cmd: "stop-task test" [Mon May 6 16:53:48 CST 2024] <<<<<< finish DM-REPLACE_ERROR >>>>>> [Mon May 6 16:53:48 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 1 [Mon May 6 16:53:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [adjust_gtid] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:53:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process worker1 exit... process worker1 already exit VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [incremental_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:53:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time run tidb sql failed 4-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:53:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:53:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... process worker2 already exit dmctl test cmd: "list-member --worker" got=2 expected=2 [Mon May 6 16:53:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:53:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:53:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source1.yaml" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:53:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 5-th time, retry later try to isolate leader dm-master2 kill dm-master2 rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:53:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "pause-task test" [Mon May 6 16:53:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "stop-task test" [Mon May 6 16:53:52 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING pessimistic >>>>>> [Mon May 6 16:53:52 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master2 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" run tidb sql failed 6-th time, retry later wait process dm-master2 exit... process dm-master2 already exit [Mon May 6 16:53:53 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> show_ddl_locks_no_locks dmctl test cmd: "shard-ddl-lock test" dmctl test cmd: "show-ddl-locks test" got=2 expected=2 try to isolate follower dm-master1 kill dm-master1 dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" dmctl test cmd: "pause-task test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" [Mon May 6 16:53:54 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING optimistic >>>>>> [Mon May 6 16:53:54 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" got=3 expected=3 got=2 expected=2 dmctl master_addr dmctl test cmd: "list-member" dmctl test cmd: "list-member" dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml --remove-meta" got=2 expected=2 check diff successfully wait process dm-master1 exit... dmctl test cmd: "resume-task test" got=2 expected=2 finish incremental_data check diff 1 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" dmctl test cmd: "list-member" [Mon May 6 16:53:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" check diff successfully 40 11 11 66 dmctl test cmd: "stop-task test" [Mon May 6 16:53:55 CST 2024] <<<<<< finish DM-MULTIPLE_ROWS >>>>>> wait process dm-master1 exit... process dm-master1 already exit [Mon May 6 16:53:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) try to isolate dm-worker1 kill dm-worker1 got=3 expected=3 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:53:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:53:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker exit... wait process dm-worker1 exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm) 2798392320 bytes in 9.53 secs (293615273 bytes/sec) [Pipeline] { [Pipeline] sh rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:53:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] // timeout rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" wait process dm-worker exit... process dm-worker already exit [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:53 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:53 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:53 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:53 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:53 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:53 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:53 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:53 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 16:53:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> try to isolate dm-worker2 kill dm-worker2 dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" [Mon May 6 16:53:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Mon May 6 16:53:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { + '[' TLS_GROUP == G10 ']' + echo 'run G10 test' run G10 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ dmctl test cmd: "stop-task test" [Mon May 6 16:53:58 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> [Mon May 6 16:53:58 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" + make dm_integration_test_in_group GROUP=G10 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:53:58 CST 2024] <<<<<< start DM-CAUSALITY >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker2 exit... [Pipeline] sh dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "query-status test" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G10 Run cases: start_task print_status http_apis new_relay all_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) got=2 expected=2 dmctl test cmd: "query-status test" [Pipeline] // timeout [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 376K drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 16:53 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 6 16:46 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 6 16:53 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 6 16:46 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:53 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 .github -rw-r--r-- 1 jenkins jenkins 985 May 6 16:46 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 6 16:46 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 6 16:46 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 6 16:46 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 6 16:46 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 6 16:46 Makefile.engine -rw-r--r-- 1 jenkins jenkins 712 May 6 16:46 OWNERS -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 6 16:46 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 6 16:46 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 6 16:46 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 6 16:46 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 bin drwxr-sr-x 17 jenkins jenkins 4.0K May 6 16:53 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 6 16:53 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 6 16:53 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 6 16:53 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 6 16:53 engine -rwxr-xr-x 1 jenkins jenkins 26K May 6 16:46 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 examples -rw-r--r-- 1 jenkins jenkins 20K May 6 16:46 go.mod -rw-r--r-- 1 jenkins jenkins 151K May 6 16:46 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 6 16:53 metrics drwxr-sr-x 45 jenkins jenkins 4.0K May 6 16:53 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 6 16:53 tests -rw-r--r-- 1 jenkins jenkins 582 May 6 16:46 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 6 16:53 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" wait process dm-worker2 exit... process dm-worker2 already exit [Mon May 6 16:53:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> dmctl test cmd: "pause-task test" got=3 expected=3 dmctl test cmd: "resume-task test" got=4 expected=4 run tidb sql failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" + '[' TLS_GROUP == G08 ']' + echo 'run G08 test' run G08 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ dmctl test cmd: "query-status test" got=1 expected=1 got=3 expected=3 + make dm_integration_test_in_group GROUP=G08 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:00 CST 2024] <<<<<< finish DM-CAUSALITY >>>>>> [Mon May 6 16:54:00 CST 2024] <<<<<< start DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "list-member" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit success success [Mon May 6 16:54:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:54:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G08 Run cases: shardddl3 shardddl3_1 shardddl4 shardddl4_1 sharding sequence_sharding ... got=2 expected=2 rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [start_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later got=3 expected=3 [Mon May 6 16:54:02 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> [Mon May 6 16:54:02 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" [Pipeline] } rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [shardddl3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh... Verbose mode = false rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/source1.yaml" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] } [Pipeline] // timeout [Pipeline] sh rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:04 CST 2024] <<<<<< finish DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> [Mon May 6 16:54:04 CST 2024] <<<<<< start DM-UpdateBARule optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again [Pipeline] // timeout dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner [Pipeline] sh dmctl test cmd: "query-status test" [Mon May 6 16:54:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time + '[' TLS_GROUP == G09 ']' + echo 'run G09 test' run G09 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G09 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source1.yaml" [Mon May 6 16:54:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time + '[' TLS_GROUP == TLS_GROUP ']' + echo 'run tls test' run tls test + echo 'copy mysql certs' copy mysql certs + sudo mkdir -p /var/lib/mysql + sudo chmod 777 /var/lib/mysql + sudo chown -R 1000:1000 /var/lib/mysql + sudo cp -r /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ca-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ca.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/client-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/client-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/private_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/public_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/server-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/server-key.pem /var/lib/mysql/ + sudo chown -R 1000:1000 /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem + ls -alh /var/lib/mysql/ total 40K drwxrwxrwx 2 jenkins jenkins 4.0K May 6 16:54 . drwxr-xr-x 1 root root 4.0K May 6 16:54 .. -rw------- 1 jenkins jenkins 1.7K May 6 16:54 ca-key.pem -rw-r--r-- 1 jenkins jenkins 1.1K May 6 16:54 ca.pem -rw-r--r-- 1 jenkins jenkins 1.1K May 6 16:54 client-cert.pem -rw------- 1 jenkins jenkins 1.7K May 6 16:54 client-key.pem -rw------- 1 jenkins jenkins 1.7K May 6 16:54 private_key.pem -rw-r--r-- 1 jenkins jenkins 451 May 6 16:54 public_key.pem -rw-r--r-- 1 jenkins jenkins 1.1K May 6 16:54 server-cert.pem -rw------- 1 jenkins jenkins 1.7K May 6 16:54 server-key.pem + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt dmctl test cmd: "query-status test" got=2 expected=2 + make dm_integration_test_in_group GROUP=TLS_GROUP /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector dmctl test cmd: "query-status test" got=3 expected=3 /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "list-member" [Mon May 6 16:54:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/shardddl1/task.yaml" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G09 Run cases: import_v10x sharding2 ha ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh TLS_GROUP dmctl test cmd: "list-member" dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Mon May 6 16:54:07 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP pessimistic >>>>>> [Mon May 6 16:54:07 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" Run cases: tls ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "show-ddl-locks" got=1 expected=1 restart dm-master [Mon May 6 16:54:08 CST 2024] <<<<<< start DM-071 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-06 10:54:05'" success success finish running run safe mode recover case 1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master exit... version() 5.7.26-log version() 8.0.21 start running case: [import_v10x] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit source1 master status, dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz-bin.000001, 1049, b0f4eaaa-0b85-11ef-9467-2e8f8a5c6ab0:1-4 source2 master status, dm-it-79826caa-00c5-4373-a614-1e68c5baf4c3-mq9bv-wcxxz-bin.000001, 1158, b159ae07-0b85-11ef-9d1d-2e8f8a5c6ab0:1-4 dmctl test cmd: "stop-task test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 [Mon May 6 16:54:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" start running case: [tls] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "resume-task test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=4 expected=4 dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:54:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "pause-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "resume-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" check diff successfully [2024/05/06 16:52:58.348 +08:00] [INFO] [dumpling.go:111] ["create dumpling"] [task=test] [unit=dump] [config="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"auto\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"\",\"ServerInfo\":{\"ServerType\":0,\"ServerVersion\":null,\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/06 16:52:58.497 +08:00] [INFO] [dump.go:151] ["begin to run Dump"] [task=test] [unit=dump] [conf="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"flush\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"sql\",\"ServerInfo\":{\"ServerType\":1,\"ServerVersion\":\"5.7.26-log\",\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/06 16:53:08.368 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14189] [2024/05/06 16:53:18.369 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14189] [2024/05/06 16:53:28.370 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14189] [2024/05/06 16:53:38.370 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=9415377] [2024/05/06 16:53:48.371 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=10306394] [2024/05/06 16:53:58.371 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=20038188] [2024/05/06 16:54:08.371 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=20038188] check diff 2 wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-master exit... process dm-master already exit check diff successfully dmctl test cmd: "purge-relay --filename dm-it-78976767-a1b4-484f-b9e8-871e87382bf7-s9l9x-lcl1w-bin.000003 -s mysql-replica-01" [Mon May 6 16:54:10 CST 2024] <<<<<< finish DM-071 pessimistic >>>>>> dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker2" wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-06 10:54:05'" [Mon May 6 16:54:11 CST 2024] <<<<<< start DM-073 pessimistic >>>>>> dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source2.yaml" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Mon May 6 16:54:11 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP optimistic >>>>>> [Mon May 6 16:54:11 CST 2024] <<<<<< start DM-SKIP_INCOMPATIBLE_DDL >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=2 expected=2 start task in incremental mode with zero gtid/pos [Mon May 6 16:54:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '1995-03-07 01:02:03'" [Mon May 6 16:54:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "config task test --path /tmp/dm_test/import_v10x/task.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh: line 556: hello: command not found dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "binlog skip test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:13 CST 2024] <<<<<< finish DM-073 pessimistic >>>>>> [Mon May 6 16:54:13 CST 2024] <<<<<< finish DM-SKIP_INCOMPATIBLE_DDL >>>>>> [Mon May 6 16:54:13 CST 2024] <<<<<< start DM-4202 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2037-12-12 01:02:03'" wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 2 [Mon May 6 16:54:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 16:54:14 CST 2024] <<<<<< start DM-076 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 3-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:35724" dmctl test cmd: "query-status test" check diff successfully check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:15 CST 2024] <<<<<< test case dmctl_basic success! >>>>>> start running case: [dmctl_command] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later [Mon May 6 16:54:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Mon May 6 16:54:15 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Mon May 6 16:54:15 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "show-ddl-locks" dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:54:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive exit code should be not zero exit code should be not zero [Mon May 6 16:54:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Mon May 6 16:54:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:54:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:54:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" [Mon May 6 16:54:17 CST 2024] <<<<<< finish DM-4202 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:54:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:18 CST 2024] <<<<<< finish DM-076 pessimistic >>>>>> [Mon May 6 16:54:18 CST 2024] <<<<<< start DM-076 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "show-ddl-locks" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 16:54:18 CST 2024] <<<<<< start DM-4204 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" [Mon May 6 16:54:18 CST 2024] <<<<<< finish DM-UpdateBARule optimistic >>>>>> [Mon May 6 16:54:18 CST 2024] <<<<<< start DM-RENAME_TABLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/wrong-source.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml -w worker1" rpc addr 127.0.0.1:8263 is alive check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "list-member --name worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w wrong-worker" check master alive dmctl test cmd: "list-member" rpc addr 127.0.0.1:8262 is alive check diff failed 1-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w worker1" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" check diff successfully check diff failed 1-th time, retry later wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:20 CST 2024] <<<<<< test case import_v10x success! >>>>>> start running case: [sharding2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" [Mon May 6 16:54:20 CST 2024] <<<<<< finish DM-4204 >>>>>> dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml" [Mon May 6 16:54:21 CST 2024] <<<<<< start DM-4206 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" [Mon May 6 16:54:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time test http and api interface Release Version: v7.5.1-26-g93530e277 Git Commit Hash: 93530e27720635216b5d5fcf531a48b801612005 Git Branch: HEAD UTC Build Time: 2024-05-06 03:00:14 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true rpc addr 127.0.0.1:8261 is alive { "result": true, "msg": "", "members": [ { "leader": { "msg": "", "name": "master2", "addr": "127.0.0.1:8361" } }, { "master": { "msg": "", "masters": [ { "name": "master1", "memberID": "10068700568285020701", "alive": true, "peerURLs": [ "https://127.0.0.1:8291" ], "clientURLs": [ "https://127.0.0.1:8261" ] }, { "name": "master2", "memberID": "518295880577331676", "alive": true, "peerURLs": [ "https://127.0.0.1:8292" ], "clientURLs": [ "https://127.0.0.1:8361" ] }, { "name": "master3", "memberID": "6800138986734450252", "alive": true, "peerURLs": [ "https://127.0.0.1:8293" ], "clientURLs": [ "https://127.0.0.1:8461" ] } ] } }, { "worker": { "msg": "", "workers": [ { "name": "worker1", "addr": "127.0.0.1:8262", "stage": "bound", "source": "mysql-replica-01" } ] } } ] } rpc addr 127.0.0.1:8261 is alive use common name not in 'cert-allowed-cn' should not request success wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:54:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:54:21 CST 2024] <<<<<< finish DM-076 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:22 CST 2024] <<<<<< finish DM-RENAME_TABLE pessimistic >>>>>> [Mon May 6 16:54:22 CST 2024] <<<<<< start DM-RENAME_TABLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:54:23 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 [2024/05/06 16:54:14.483 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b342e6e9-0b85-11ef-b1e8-e6a2a9a9b733.000001/dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk-bin.000001] [2024/05/06 16:54:14.501 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b342e6e9-0b85-11ef-b1e8-e6a2a9a9b733.000001/dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk-bin.000001] [2024/05/06 16:54:19.111 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b342e6e9-0b85-11ef-b1e8-e6a2a9a9b733.000001/dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk-bin.000001] [2024/05/06 16:54:19.128 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b342e6e9-0b85-11ef-b1e8-e6a2a9a9b733.000001/dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk-bin.000001] [2024/05/06 16:54:19.137 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b342e6e9-0b85-11ef-b1e8-e6a2a9a9b733.000001/dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk-bin.000001] [2024/05/06 16:54:19.157 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b342e6e9-0b85-11ef-b1e8-e6a2a9a9b733.000001/dm-it-1094b50d-db4d-4140-898d-1a0a60fdc796-t4nnf-cp8qk-bin.000001] dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "pause-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:39469 -s mysql-replica-01" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source1.yaml" [Mon May 6 16:54:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:39276 -s mysql-replica-01" wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:36408 -s mysql-replica-02" wait for rpc addr 127.0.0.1:8261 alive the 3-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:36627 -s mysql-replica-02" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:24 CST 2024] <<<<<< test case adjust_gtid success! >>>>>> start running case: [async_checkpoint_flush] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" [Mon May 6 16:54:24 CST 2024] <<<<<< finish DM-RENAME_TABLE optimistic >>>>>> [Mon May 6 16:54:24 CST 2024] <<<<<< start DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:54:24 CST 2024] <<<<<< finish DM-077 >>>>>> [Mon May 6 16:54:24 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 4-th time check diff failed 1-th time, retry later wait process tidb-server exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 5-th time wait process dm-worker.test exit... got=3 expected=3 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/async_checkpoint_flush/source1.yaml" wait process tidb-server exit... process tidb-server already exit [Mon May 6 16:54:26 CST 2024] <<<<<< finish DM-077 >>>>>> dmctl test cmd: "stop-task test" [Mon May 6 16:54:26 CST 2024] <<<<<< finish DM-4206 pessimistic >>>>>> [Mon May 6 16:54:26 CST 2024] <<<<<< start DM-4206 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-task.yaml " got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 6-th time process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit success success [Mon May 6 16:54:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:54:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:54:27 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 7-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully PID of insert_data is 2109 check sync diff for the increment replication check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:41438 -s mysql-replica-01" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:41245 -s mysql-replica-01" check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:38566 -s mysql-replica-02" dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:38785 -s mysql-replica-02" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 8-th time check sync diff for the increment replication check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 16:54:29 CST 2024] <<<<<< finish DM-4206 optimistic >>>>>> check diff failed 1-th time, retry later wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:30 CST 2024] <<<<<< finish DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " wait for rpc addr 127.0.0.1:8261 alive the 9-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:30 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 10-th time dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner dmctl test cmd: "query-status test" wait process dm-master exit... process dm-master already exit [Mon May 6 16:54:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Mon May 6 16:54:20 CST 2024] <<<<<< finish test_kill_master_in_sync >>>>>> [Mon May 6 16:54:20 CST 2024] <<<<<< start test_kill_worker_in_sync >>>>>> [Mon May 6 16:54:20 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master cluster [Mon May 6 16:54:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Mon May 6 16:54:31 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Mon May 6 16:54:31 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 11-th time check diff successfully kill dm-worker1 rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:43214" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Mon May 6 16:54:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:40724" dmctl test cmd: "query-status test" wait process dm-worker1 exit... got=3 expected=3 dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:33 CST 2024] <<<<<< finish DM-078 >>>>>> [Mon May 6 16:54:33 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" [Mon May 6 16:54:33 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Mon May 6 16:54:33 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 12-th time check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" got=1 expected=1 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 13-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:54:34 CST 2024] <<<<<< start DM-RemoveLock pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:44990" [Mon May 6 16:54:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:42663" check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker2 exit... dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" success success finish running run safe mode recover case 2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 14-th time [Mon May 6 16:54:35 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> [Mon May 6 16:54:35 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" wait process dm-worker2 exit... process dm-worker2 already exit [Mon May 6 16:54:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 15-th time wait process dm-worker.test exit... wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:36 CST 2024] <<<<<< finish DM-078 >>>>>> dmctl test cmd: "query-status test2" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:47024" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:54:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:44602" dmctl test cmd: "query-status test" wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 16-th time [Mon May 6 16:54:37 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:54:38 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Mon May 6 16:54:38 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait for port 8262 offline the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 1-th time, retry later check diff successfully dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchTargetDoTablesFailed=return(1152) wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 17-th time dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 0\/0\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait for port 8262 offline the 2-th time got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 16:54:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 18-th time [Mon May 6 16:54:39 CST 2024] <<<<<< finish DM-079 >>>>>> [Mon May 6 16:54:39 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:49058" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:46541" [Mon May 6 16:54:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 3 [Mon May 6 16:54:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 19-th time [Mon May 6 16:54:40 CST 2024] <<<<<< finish DM-RemoveLock pessimistic >>>>>> [Mon May 6 16:54:40 CST 2024] <<<<<< start DM-RemoveLock optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:54:40 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again [Mon May 6 16:54:41 CST 2024] <<<<<< finish DM-079 >>>>>> rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 20-th time dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:54:41 CST 2024] <<<<<< start DM-4209 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 1-th time, retry later use sync_diff_inspector to check increment data check diff successfully [Mon May 6 16:54:42 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source kill dm-worker1 [Mon May 6 16:54:42 CST 2024] <<<<<< start DM-080 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" check diff successfully rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/0/source1.yaml" rpc addr 127.0.0.1:8261 check alive timeout check data check diff successfully ============================== test_master_ha_when_enable_tidb_and_only_ca_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" dmctl test cmd: "query-status test" run tidb sql failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" run tidb sql failed 1-th time, retry later check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" wait process dm-worker1 exit... process dm-worker1 already exit start worker3 [Mon May 6 16:54:44 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=2 check sync diff for the increment replication check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" got=3 expected=3 check diff failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker3" wait process dm-worker.test exit... wait process dm-master.test exit... [Mon May 6 16:54:45 CST 2024] <<<<<< finish DM-4209 pessimistic >>>>>> [Mon May 6 16:54:45 CST 2024] <<<<<< start DM-4209 optimistic >>>>>> dmctl test cmd: "validation status test" wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" got=4 expected=4 got=1 expected=1 dmctl test cmd: "validation status --table-stage running test" got=4 expected=4 got=0 expected=0 dmctl test cmd: "validation status --table-stage stopped test" got=2 expected=2 kill dm-worker2 wait process dm-worker.test exit... process dm-worker.test already exit got=2 expected=2 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=0 expected=1 command: validation show-error --error all test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" check diff successfully wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" wait process dm-worker2 exit... wait process dm-master.test exit... wait process dm-master.test exit... run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test" [Mon May 6 16:54:48 CST 2024] <<<<<< finish DM-080 pessimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:54:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker2 exit... process dm-worker2 already exit start worker4 [Mon May 6 16:54:48 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:54:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... [Mon May 6 16:54:49 CST 2024] <<<<<< start DM-081 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status --table-stage stopped" dmctl test cmd: "validation status --table-stage start test" dmctl test cmd: "validation show-error --error resolved test" dmctl test cmd: "validation show-error --error all" dmctl test cmd: "validation ignore-error test 100 --all" dmctl test cmd: "validation resolve-error test 100 --all" dmctl test cmd: "validation clear-error test 100 --all" dmctl test cmd: "validation ignore-error test 100 101" dmctl test cmd: "validation resolve-error test 100 101" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 3650 0 3650 0 0 476k 0 --:--:-- --:--:-- --:--:-- 509k dmctl test cmd: "stop-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:49 CST 2024] <<<<<< test case sharding2 success! >>>>>> start running case: [ha] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM worker and master [Mon May 6 16:54:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master1.toml >>>>>> [Mon May 6 16:54:49 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:18262 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive query-status from all dm-master dmctl test cmd: "query-status test" wait process dm-master.test exit... dmctl test cmd: "validation clear-error test 100 101" dmctl test cmd: "validation ignore-error test error-id" dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:54:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=3 expected=3 dmctl test cmd: "query-status test" dmctl test cmd: "validation resolve-error test error-id" dmctl test cmd: "validation clear-error test error-id" [Mon May 6 16:54:50 CST 2024] <<<<<< finish DM-4209 optimistic >>>>>> dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=3 expected=3 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" dmctl test cmd: "validation ignore-error test" dmctl test cmd: "validation resolve-error test" dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" got=3 expected=3 got=2 expected=2 dmctl test cmd: "validation clear-error test" dmctl test cmd: "validation ignore-error" dmctl test cmd: "validation resolve-error" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" dmctl test cmd: "validation clear-error" dmctl test cmd: "validation ignore-error non-exist-task-name 1" dmctl test cmd: "validation resolve-error non-exist-task-name 1" dmctl test cmd: "validation clear-error non-exist-task-name 1" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "validation resolve-error test 1" dmctl test cmd: "validation status test" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" [Mon May 6 16:54:51 CST 2024] <<<<<< start DM-4211 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "validation show-error --error unprocessed test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error test" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test 2" dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8262 is alive wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" got=1 expected=1 dmctl test cmd: "validation clear-error test 1" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation clear-error test 2" dmctl test cmd: "validation status test" wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation stop test" got=1 expected=1 dmctl test cmd: "validation status test" dmctl test cmd: "binlog replace test --binlog-pos dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:54386 alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Mon May 6 16:54:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker1.toml >>>>>> got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=3 expected=3 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 60744 0 60744 0 0 25.6M 0 --:--:-- --:--:-- --:--:-- 28.9M % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 69157 0 69157 0 0 19.1M 0 --:--:-- --:--:-- --:--:-- 21.9M dmctl test cmd: "validation show-error --error all test" got=1 expected=1 got=2 expected=2 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:53 CST 2024] <<<<<< finish DM-081 pessimistic >>>>>> got=1 expected=1 dmctl test cmd: "validation ignore-error test 3" dmctl test cmd: "validation resolve-error test 4" [Mon May 6 16:54:53 CST 2024] <<<<<< finish DM-4211 >>>>>> dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation clear-error test --all" wait process dm-worker.test exit... dmctl test cmd: "validation status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls wait process dm-worker.test exit... process dm-worker.test already exit success success [Mon May 6 16:54:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:54:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 got=2 expected=2 dmctl test cmd: "validation status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha/source1.yaml" [Mon May 6 16:54:54 CST 2024] <<<<<< start DM-082 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 16:54:54 CST 2024] <<<<<< start DM-4213 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "stop-task test" clean up data dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/dm-task.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 [Mon May 6 16:54:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:54:55 CST 2024] <<<<<< test case incremental_mode success! >>>>>> start running case: [initial_unit] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "validation status test" check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" dmctl test cmd: "query-status test" got=2 expected=2 --> validation resolve-error --all 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 dmctl test cmd: "stop-task test" [Mon May 6 16:54:56 CST 2024] <<<<<< finish DM-4213 >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha/source2.yaml" failpoint=github.com/pingcap/tiflow/dm/syncer/LoadCheckpointFailed=return(1152) [Mon May 6 16:54:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:54:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:54:58 CST 2024] <<<<<< finish DM-082 pessimistic >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner process dm-master.test already exit wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/0/source1.yaml" start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" wait process dm-worker.test exit... [Mon May 6 16:54:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Mon May 6 16:54:59 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Mon May 6 16:54:59 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 [Mon May 6 16:54:58 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time [Mon May 6 16:54:59 CST 2024] <<<<<< start DM-085 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... rpc addr 127.0.0.1:8461 is alive ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 1-th time, got value: '' check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh: line 29: 2109 Terminated insert_data 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 2-th time, got value: '' wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success success finish running run safe mode recover case 3 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" resume task will also initial failed dmctl test cmd: "resume-task test" wait process dm-worker.test exit... ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 3-th time, got value: '' wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:02 CST 2024] <<<<<< test case handle_error success! >>>>>> start running case: [handle_error_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh... Verbose mode = false rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive wait process dm-worker.test exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8461 is alive [Mon May 6 16:55:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchAllDoTablesFailed=return(1152) [Mon May 6 16:55:03 CST 2024] <<<<<< finish DM-085 pessimistic >>>>>> [Mon May 6 16:55:03 CST 2024] <<<<<< start DM-085 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:55:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:03 CST 2024] <<<<<< test case async_checkpoint_flush success! >>>>>> start running case: [binlog_parse] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive metric dm_master_worker_state{worker="worker1"} has invalid value start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml " rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:55:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker1.toml >>>>>> wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-only-ca.yaml" wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:55:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source1.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 16:55:05 CST 2024] <<<<<< finish DM-085 optimistic >>>>>> % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 96136 0 --:--:-- --:--:-- --:--:-- 104k check diff failed 1-th time, retry later wait process dm-worker.test exit... start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/binlog_parse/source1.yaml" prepare data start task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:55:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 16:55:06 CST 2024] <<<<<< start DM-086 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:55:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 4 [Mon May 6 16:55:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-2.yaml --remove-meta=true" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source2.yaml" dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" check full phase check diff successfully dmctl test cmd: "validation start test" prepare incremental data check incremental phase check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" check diff successfully dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" start dm-worker3 and kill dm-worker2 dmctl test cmd: "query-status test" got=1 expected=1 reset go failpoints, and need restart dm-worker, then start task again rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 16:55:08 CST 2024] <<<<<< start DM-4215 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 16:55:08 CST 2024] <<<<<< finish DM-086 pessimistic >>>>>> [Mon May 6 16:55:08 CST 2024] <<<<<< start DM-086 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "validation status test" dmctl test cmd: "query-status test2" check data wait process dm-worker.test exit... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 90163 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused got=1 expected=1 got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 0\/0\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=2 expected=2 got=1 expected=1 got=1 expected=1 prepare incremental data 2 dmctl test cmd: "validation start test" check incremental phase 2 check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff successfully check diff successfully ============================== test_worker_handle_multi_tls_tasks success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker2 exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:55:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit [Mon May 6 16:55:09 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" wait process dm-worker.test exit... process dm-worker.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" [Mon May 6 16:55:10 CST 2024] <<<<<< finish DM-086 optimistic >>>>>> dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait process dm-master.test exit... rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:55:11 CST 2024] <<<<<< start DM-094 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml" wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:55:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... check diff successfully dmctl test cmd: "validation show-error test" got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:13 CST 2024] <<<<<< test case binlog_parse success! >>>>>> start running case: [case_sensitive] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... [Mon May 6 16:55:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:55:14 CST 2024] <<<<<< finish DM-094 pessimistic >>>>>> [Mon May 6 16:55:14 CST 2024] <<<<<< start DM-094 optimistic >>>>>> % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 117 100 117 0 0 166k 0 --:--:-- --:--:-- --:--:-- 114k {"connections":0,"version":"8.0.11-TiDB-v7.5.1-48-g066b97f741","git_hash":"066b97f741e6fbef0740e073940cea08d13b3f43"}dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task-2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:55:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation resolve-error test --all" dmctl test cmd: "validation show-error test" wait process dm-master.test exit... got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 --> validation ignore-error --all 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:55:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" rpc addr 127.0.0.1:8262 is alive wait process dm-master.test exit... [Mon May 6 16:55:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" check diff successfully dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml" dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... process dm-master.test already exit run tidb sql failed 2-th time, retry later wait process dm-worker.test exit... [Mon May 6 16:55:16 CST 2024] <<<<<< finish DM-RemoveLock optimistic >>>>>> wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Mon May 6 16:55:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls wait for the task to be scheduled and keep paused HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive resume task before kill and restart dm-worker dmctl test cmd: "resume-task test" wait process dm-worker.test exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... start dm-worker2 and kill dm-worker3 [Mon May 6 16:55:19 CST 2024] <<<<<< finish DM-094 optimistic >>>>>> dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit success success [Mon May 6 16:55:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:55:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... run tidb sql failed 3-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker3 exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 [Mon May 6 16:55:20 CST 2024] <<<<<< start DM-095 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit check diff successfully wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker3 exit... process dm-worker3 already exit [Mon May 6 16:55:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> [Mon May 6 16:55:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait pattern dm-worker1.toml exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" wait pattern dm-worker1.toml exit... dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" got=2 expected=2 [Mon May 6 16:55:22 CST 2024] <<<<<< start DM-RestartMaster pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait pattern dm-worker1.toml exit... wait process dm-worker.test exit... run tidb sql failed 4-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "pause-task test" wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Mon May 6 16:55:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:55:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 1-th time, retry later got=1 expected=1 wait pattern dm-worker2.toml exit... failpoint=github.com/pingcap/tiflow/dm/syncer/dbconn/GetMasterStatusFailed=return(1152) [Mon May 6 16:55:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait pattern dm-worker2.toml exit... run tidb sql failed 5-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner wait pattern dm-worker2.toml exit... [Mon May 6 16:55:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Mon May 6 16:55:24 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Mon May 6 16:55:24 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-master.test exit... wait pattern dm-worker2.toml exit... dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Mon May 6 16:55:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:55:25 CST 2024] <<<<<< finish DM-095 pessimistic >>>>>> [Mon May 6 16:55:25 CST 2024] <<<<<< start DM-095 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Mon May 6 16:55:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive run tidb sql failed 6-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-master exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test2" got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/1/source1.yaml" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:55:27 CST 2024] <<<<<< finish DM-095 optimistic >>>>>> success success finish running run safe mode recover case 4 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master exit... process dm-master already exit start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:28 CST 2024] <<<<<< start test_COMMIT_in_QueryEvent >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit run tidb sql failed 7-th time, retry later [Mon May 6 16:55:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/start_task/1/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... check diff successfully dmctl test cmd: "validation status test" [Mon May 6 16:55:28 CST 2024] <<<<<< start DM-096 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" check data check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:55:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start dm-worker2 and kill dm-worker1 dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "pause-task test" run tidb sql failed 8-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/start_task/1/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "resume-task test" got=3 expected=3 wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Mon May 6 16:55:30 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker1 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/start_task/1/dm-task.yaml --remove-meta=true" resume task will also initial failed dmctl test cmd: "resume-task test" wait process dm-worker.test exit... rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker3" dmctl test cmd: "list-member --name worker3" got=1 expected=1 query-status from all dm-master dmctl test cmd: "query-status test" Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 wait process dm-worker1 exit... process dm-worker1 already exit mysql_ssl_setup at=/var/lib/mysql/ [Mon May 6 16:55:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/tls/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 join new dm-master and query-status [Mon May 6 16:55:32 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:55:33 CST 2024] <<<<<< finish DM-096 pessimistic >>>>>> [Mon May 6 16:55:33 CST 2024] <<<<<< start DM-096 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:55:33 CST 2024] <<<<<< finish DM-RestartMaster pessimistic >>>>>> [Mon May 6 16:55:33 CST 2024] <<<<<< start DM-RestartMaster optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 resume task to worker2 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "query-status test" got=3 expected=3 check diff successfully ============================== test_worker_ha_when_enable_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later dmctl test cmd: "validation show-error test" dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" dmctl test cmd: "stop-task test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:55:34 CST 2024] <<<<<< finish DM-4215 pessimistic >>>>>> [Mon May 6 16:55:34 CST 2024] <<<<<< start DM-4215 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... check diff successfully <<<<<< test_COMMIT_in_QueryEvent success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Mon May 6 16:55:35 CST 2024] <<<<<< finish DM-096 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:55:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive check diff failed 1-th time, retry later [Mon May 6 16:55:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Mon May 6 16:55:36 CST 2024] <<<<<< start DM-097 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" use sync_diff_inspector to check increment data check diff successfully [Mon May 6 16:55:32 CST 2024] <<<<<< finish test_isolate_master_and_worker >>>>>> 5 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=2 command: query-status test "ErrScope": "downstream" count: 1 != expected: 2, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=2 expected=2 got=1 expected=1 restart dm-master wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit wait process dm-master exit... check diff successfully wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test --all" dmctl test cmd: "validation show-error test" got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task-safe-mode-duration.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... check diff successfully wait process dm-master exit... process dm-master already exit wait process dm-master exit... wait process dm-master.test exit... wait process dm-master.test exit... [Mon May 6 16:55:39 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time wait process dm-master.test exit... got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:39 CST 2024] <<<<<< test case many_tables success! >>>>>> start running case: [online_ddl] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:39 CST 2024] <<<<<< test case start_task success! >>>>>> start running case: [print_status] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh... Verbose mode = false +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status ++ pwd + cur=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../_utils/test_prepare ++ source env_variables +++ MYSQL_HOST1=127.0.0.1 +++ MYSQL_HOST2=127.0.0.1 +++ TIDB_HOST=127.0.0.1 +++ MYSQL_PORT1=3306 +++ MYSQL_PORT2=3307 +++ MYSQL_PASSWORD1=123456 +++ MYSQL_PASSWORD2=123456 +++ TIDB_PASSWORD=123456 +++ TIDB_PORT=4000 +++ MASTER_PORT=8261 +++ MASTER_PEER_PORT=8291 +++ MASTER_PORT1=8261 +++ MASTER_PORT2=8361 +++ MASTER_PORT3=8461 +++ MASTER_PORT4=8561 +++ MASTER_PORT5=8661 +++ MASTER_PORT6=8761 +++ MASTER_PEER_PORT1=8291 +++ MASTER_PEER_PORT2=8292 +++ MASTER_PEER_PORT3=8293 +++ MASTER_PEER_PORT4=8294 +++ MASTER_PEER_PORT5=8295 +++ MASTER_PEER_PORT6=8296 +++ WORKER1_PORT=8262 +++ WORKER2_PORT=8263 +++ WORKER3_PORT=8264 +++ WORKER4_PORT=18262 +++ WORKER5_PORT=18263 +++ WORKER1_NAME=worker1 +++ WORKER2_NAME=worker2 +++ SOURCE_ID1=mysql-replica-01 +++ SOURCE_ID2=mysql-replica-02 +++ RESET_MASTER=true +++ VERBOSE=false ++ '[' true = true ']' ++ run_sql 'RESET MASTER' 3306 123456 ++ run_sql 'RESET MASTER' 3307 123456 + WORK_DIR=/tmp/dm_test/print_status + cleanup_data print_status + rm -rf /tmp/dm_test/print_status + mkdir /tmp/dm_test/print_status + for target_db in '"$@"' + run_sql 'drop database if exists `print_status`' 4000 123456 + run_sql 'drop database if exists dm_meta' 4000 123456 + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=0 + echo '0 dm-master alive' 0 dm-master alive + pkill -hup dm-master.test + true ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=0 + echo '0 dm-worker alive' 0 dm-worker alive + pkill -hup dm-worker.test + true ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test process dm-master.test already exit + wait_process_exit dm-worker.test process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + run + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.prepare.sql 127.0.0.1 3306 123456 + inject_points=("github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)" "github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)" "github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)") ++ join_string ';' 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)' 'github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)' 'github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' ++ local 'IFS=;' ++ shift ++ echo 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + GO_FAILPOINTS='github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-worker1.toml /tmp/dm_test/print_status/dm-worker1.toml + sed -i s%placeholder%/tmp/dm_test/print_status/relay_by_worker%g /tmp/dm_test/print_status/dm-worker1.toml + run_dm_master /tmp/dm_test/print_status/master 8261 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml [Mon May 6 16:55:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_master_online 127.0.0.1:8261 wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status test" wait process dm-master.test exit... got=3 expected=3 wait process dm-master exit... process dm-master already exit [Mon May 6 16:55:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... [Mon May 6 16:55:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:41 CST 2024] <<<<<< test case ha_cases3_1 success! >>>>>> start running case: [ha_master] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit start DM worker and master [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "stop-task test" [Mon May 6 16:55:41 CST 2024] <<<<<< finish DM-4215 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive + run_dm_worker /tmp/dm_test/print_status/worker1 8262 /tmp/dm_test/print_status/dm-worker1.toml [Mon May 6 16:55:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/print_status/dm-worker1.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_worker_online 127.0.0.1:8262 wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source1.yaml" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 16:55:42 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive + cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/source1.yaml /tmp/dm_test/print_status/source1.yaml + sed -i '/relay-binlog-name/i\relay-dir: /tmp/dm_test/print_status/worker1/relay_log' /tmp/dm_test/print_status/source1.yaml + dmctl_operate_source create /tmp/dm_test/print_status/source1.yaml mysql-replica-01 + operate=create + source_toml=/tmp/dm_test/print_status/source1.yaml + source_id=mysql-replica-01 + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'operate-source create /tmp/dm_test/print_status/source1.yaml' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "operate-source create /tmp/dm_test/print_status/source1.yaml" got=1 expected=1 dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit + dmctl_start_task_standalone + '[' 0 -ge 2 ']' + remove_meta= + '[' 0 -ge 1 ']' + task_conf=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml ' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml " check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Mon May 6 16:55:43 CST 2024] <<<<<< finish DM-097 pessimistic >>>>>> [Mon May 6 16:55:43 CST 2024] <<<<<< start DM-097 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 16:55:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:5043 alter table handle_error.tb1 add column c varchar(20);" [Mon May 6 16:55:44 CST 2024] <<<<<< finish DM-RestartMaster optimistic >>>>>> [Mon May 6 16:55:44 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check diff successfully + ls /tmp/dm_test/print_status/relay_by_worker/worker1/b1de1fa2-0b85-11ef-ae00-026c1646ad2a.000001 /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/b1de1fa2-0b85-11ef-ae00-026c1646ad2a.000001: dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9-bin.000001 relay.meta + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.increment.sql 127.0.0.1 3306 123456 + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check diff failed 1-th time, retry later got=3 expected=3 dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:5548 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-worker.test exit... [Mon May 6 16:55:45 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Mon May 6 16:55:45 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully [Mon May 6 16:55:45 CST 2024] <<<<<< START DM-MASTER on port 8761, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master6.toml >>>>>> wait for rpc addr 127.0.0.1:8761 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully finish running safe mode duration case 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit check diff failed 1-th time, retry later [Mon May 6 16:55:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:55:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8761 is alive dmctl test cmd: "query-status test" dmctl test cmd: "start-task /tmp/dm_test/online_ddl/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 wait process dm-master.test exit... dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:6850 alter table handle_error.tb1 add column c varchar(20);" check diff successfully + check_log_contains /tmp/dm_test/print_status/worker1/log/dm-worker.log 'enable safe-mode because of task initialization.*duration=1m0s' + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=1 + echo '1 dm-master alive' 1 dm-master alive + pkill -hup dm-master.test ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=1 + echo '1 dm-worker alive' 1 dm-worker alive + pkill -hup dm-worker.test ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:7521 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive got=3 expected=3 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8461 is alive rpc addr 127.0.0.1:8561 is alive rpc addr 127.0.0.1:8661 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source1.yaml" [Mon May 6 16:55:47 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> [Mon May 6 16:55:47 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... use sync_diff_inspector to check full dump data check diff successfully alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff successfully check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source2.yaml" wait process dm-master exit... wait process dm-master.test exit... process dm-master.test already exit + wait_process_exit dm-worker.test start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" check diff successfully dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master exit... process dm-master already exit [Mon May 6 16:55:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:8915 alter table handle_error.tb1 add column c varchar(20);" [Mon May 6 16:55:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:9494 alter table handle_error.tb1 add column c varchar(20);" wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "stop-task test" check diff successfully [Mon May 6 16:55:50 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Mon May 6 16:55:50 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:51 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" kill dm-master1 got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:10980 alter table handle_error.tb1 add column c varchar(20);" wait process dm-worker.test exit... process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + check_print_status + i=0 + '[' 0 -lt 3 ']' ++ grep 'dm-worker exit' /tmp/dm_test/print_status/worker1/log/dm-worker.log + exit_log='[2024/05/06 16:55:49.789 +08:00] [INFO] [main.go:93] ["dm-worker exit"]' + '[' '[2024/05/06 16:55:49.789 +08:00] [INFO] [main.go:93] ["dm-worker exit"]' == 'not found' ']' + break + '[' 0 -ge 3 ']' + echo 'checking print status' checking print status + dump_status_file=/tmp/dm_test/print_status/worker1/log/dump_status.log + grep -o 'progress status of dumpling' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=1 + '[' 1 -ge 1 ']' + grep -o '\[bps=0' /tmp/dm_test/print_status/worker1/log/dm-worker.log + true ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=0 + '[' 0 -eq 0 ']' + status_file=/tmp/dm_test/print_status/worker1/log/loader_status.log + grep -oP '\[unit=lightning-load\] \[IsCanceled=false\] \[finished_bytes=59674\] \[total_bytes=59674\] \[progress=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/loader_status.log ++ awk '{print $1}' + status_count=1 + '[' 1 -eq 1 ']' + grep 'current speed (bytes / seconds)' /tmp/dm_test/print_status/worker1/log/dm-worker.log + grep -vq '"current speed (bytes / seconds)"=0' + echo 'check load unit print status success' check load unit print status success + status_file2=/tmp/dm_test/print_status/worker1/log/syncer_status.log + grep -oP '\[total_rows=[0-9]+\] \[total_rps=[0-9]+\] \[rps=[0-9]+\] \[master_position=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/syncer_status.log ++ awk '{print $1}' + status_count2=7 + '[' 7 -ge 1 ']' + echo 'check sync unit print status success' check sync unit print status success ++ date + echo '[Mon May 6 16:55:51 CST 2024] <<<<<< test case print_status success! >>>>>>' [Mon May 6 16:55:51 CST 2024] <<<<<< test case print_status success! >>>>>> start running case: [http_apis] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 --> (fail) validation start: missing mode value dmctl test cmd: "validation start --mode" --> (fail) validation start: invalid mode dmctl test cmd: "validation start --mode xxx" --> (fail) validation start: missing start-time value dmctl test cmd: "validation start --start-time" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:11467 alter table handle_error.tb1 add column c varchar(20);" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=3 expected=3 dmctl test cmd: "stop-task test" --> (fail) validation start: invalid start-time dmctl test cmd: "validation start --start-time xx" --> (fail) validation start: without all-task and task-name dmctl test cmd: "validation start --mode full" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Mon May 6 16:55:52 CST 2024] <<<<<< finish DM-097 optimistic >>>>>> [Mon May 6 16:55:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time --> (fail) validation start: with both all-task and task-name dmctl test cmd: "validation start --all-task test" --> (fail) validation start: too many arguments dmctl test cmd: "validation start test test2" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:55:52 CST 2024] <<<<<< test case initial_unit success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Mon May 6 16:55:52 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> kill dm-master1 and kill dm-master2 [Pipeline] // cache --> (fail) validation start: non-existed subtask dmctl test cmd: "validation start xxxxx" --> (fail) validation start: non-exist source dmctl test cmd: "validation start -s xxxxx --all-task" wait process dm-master1 exit... [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } check diff successfully check diff failed 1-th time, retry later --> (fail) validation stop: without all-task and task-name dmctl test cmd: "validation stop" --> (fail) validation stop: with both all-task and task-name dmctl test cmd: "validation stop --all-task test" [Pipeline] // timeout wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:55:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] } check diff successfully wait pattern dm-worker1.toml exit... [Pipeline] // stage --> (fail) validation stop: too many arguments dmctl test cmd: "validation stop test test2" --> (fail) validation stop: non-existed subtask dmctl test cmd: "validation stop xxxxx" [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv wait pattern dm-worker1.toml exit... [Pipeline] } --> (fail) validation stop: non-exist source dmctl test cmd: "validation stop -s xxxxx --all-task" --> (fail) validation stop: stop not-enabled validator dmctl test cmd: "validation stop -s mysql-replica-01 test" --> (success) validation start: start validation without explicit mode for source 1 dmctl test cmd: "validation start -s mysql-replica-01 test" [Pipeline] // node wait process dm-master1 exit... [Pipeline] } wait process dm-master1 exit... process dm-master1 already exit kill dm-master2 wait pattern dm-worker1.toml exit... [Pipeline] // podTemplate --> (fail) validation start: start all validator with explicit mode, but 1 subtask already enabled dmctl test cmd: "validation start --mode full test" --> (fail) validation start: start validation with explicit mode for source 1 again dmctl test cmd: "validation start -s mysql-replica-01 --mode full test" [Pipeline] } [Mon May 6 16:55:53 CST 2024] <<<<<< start DM-098 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] // withEnv wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... [Pipeline] } wait process dm-master.test exit... [Mon May 6 16:55:53 CST 2024] <<<<<< start DM-4219 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" --> (fail) validation start: start all validator without explicit mode dmctl test cmd: "validation start test" --> (fail) validation stop: stop all but 1 subtask is not enabled dmctl test cmd: "validation stop test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" [Pipeline] // stage [Pipeline] } wait pattern dm-worker1.toml exit... --> (success) validation start: start validation with fast mode and start-time for source 2 dmctl test cmd: "validation start -s mysql-replica-02 --mode fast --start-time '2020-01-01 00:01:00' test" dmctl test cmd: "validation status test" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Mon May 6 16:55:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation start: start all validator of the task without explicit param again, i.e. resuming dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" wait process dm-master1 exit... process dm-master1 already exit wait process dm-master2 exit... dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation stop: stop validation of source 1 dmctl test cmd: "validation stop -s mysql-replica-01 test" dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:55:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> (success) validation stop: stop all of test dmctl test cmd: "validation stop test" dmctl test cmd: "validation status test" [Mon May 6 16:55:55 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS pessimistic >>>>>> [Mon May 6 16:55:55 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation stop: stop all of test again dmctl test cmd: "validation stop test" dmctl test cmd: "validation status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: start all of test dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive wait process dm-master2 exit... wait process dm-master2 exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 wait pattern dm-worker2.toml exit... process dm-master2 already exit source-id: mysql-replica-01\nflavor: ''\nenable-gtid: true\nenable-relay: true\nrelay-dir: /tmp/dm_test/http_apis/worker1/relay_log\nrelay-binlog-name: ''\nrelay-binlog-gtid: ''\nfrom:\n host: 127.0.0.1\n user: root\n password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=\n port: 3306\n initial cluster of dm-masters have been killed got=1 expected=1 got=1 expected=1 got=2 expected=2 now we will check whether joined masters can work normally % Total % Received % Xferd Average Speed Time Time Time Current dmctl test cmd: "stop-task test" Dload Upload Total Spent Left Speed got=0 expected=0 dmctl test cmd: "query-status test" --> (success) validation stop: multiple source dmctl test cmd: "validation stop -s mysql-replica-01 -s mysql-replica-02 test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: multiple source dmctl test cmd: "validation start -s mysql-replica-01 -s mysql-replica-02 test" wait process dm-worker1 exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 470 100 166 100 304 1291 2365 --:--:-- --:--:-- --:--:-- 2375 start task and check stage ---\nname: test\ntask-mode: all\nis-sharding: false\nmeta-schema: \"dm_meta\"\n# enable-heartbeat: true\n\ntarget-database:\n host: \"127.0.0.1\"\n port: 4000\n user: \"root\"\n password: \"\"\n\nmysql-instances:\n - source-id: \"mysql-replica-01\"\n block-allow-list: \"instance\"\n mydumper-config-name: \"global\"\n loader-config-name: \"global\"\n syncer-config-name: \"global\"\n\nblock-allow-list:\n instance:\n do-dbs: [\"http_apis\"]\n\nmydumpers:\n global:\n threads: 4\n chunk-filesize: 0\n skip-tz-utc: true\n statement-size: 100\n extra-args: \"\"\n\nloaders:\n global:\n pool-size: 16\n dir: \"./dumped_data\"\n\nsyncers:\n global:\n worker-count: 16\n batch: 100\n HTTP 127.0.0.1:8261/apis/v1alpha1/status/test-task is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" wait pattern dm-worker2.toml exit... wait process dm-master2 exit... process dm-master2 already exit got=2 expected=2 check diff failed 1-th time, retry later wait pattern dm-worker2.toml exit... master4 [Mon May 6 16:55:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 742 0 0 100 742 0 740 0:00:01 0:00:01 --:--:-- 740 100 950 100 208 100 742 191 683 0:00:01 0:00:01 --:--:-- 683 check diff successfully wait for port 8262 offline the 1-th time wait process dm-master.test exit... wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Mon May 6 16:55:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 16:55:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2-no-validator.yaml --remove-meta" wait for rpc addr 127.0.0.1:8262 alive the 2-th time wait for port 8262 offline the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-task test" [Mon May 6 16:55:58 CST 2024] <<<<<< finish DM-4219 pessimistic >>>>>> [Mon May 6 16:55:58 CST 2024] <<<<<< start DM-4219 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 283k 0 --:--:-- --:--:-- --:--:-- 301k get sub task configs % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2299 0 2299 0 0 552k 0 --:--:-- --:--:-- --:--:-- 748k pause task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 194 100 183 100 11 14685 882 --:--:-- --:--:-- --:--:-- 14076 100 194 100 183 100 11 14589 876 --:--:-- --:--:-- --:--:-- 14076 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml" [Mon May 6 16:55:58 CST 2024] <<<<<< finish DM-098 pessimistic >>>>>> 2024-05-06 16:55:58 ERROR Error 1146: Table 'online_ddl._gho_t3_ghc' doesn't exist alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 wait process dm-master.test exit... ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=1 expected=1 --> (fail) validation update: on non-existed validator dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid adaa546c-0b85-11ef-838e-422a3ffab8f0:1-999" --> (success) validation start: start all tasks dmctl test cmd: "validation start --all-task --mode fast" wait for rpc addr 127.0.0.1:8262 alive the 3-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1960 100 1960 0 0 300k 0 --:--:-- --:--:-- --:--:-- 319k resume task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 195 100 184 100 11 8827 527 --:--:-- --:--:-- --:--:-- 9200 check diff successfully check diff successfully check diff successfully [Mon May 6 16:55:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:55:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time --> (fail) validation update: missing task-name dmctl test cmd: "validation update" --> (fail) validation update: multi tasks dmctl test cmd: "validation update test test2" check diff successfully dmctl test cmd: "stop-task test" use sync_diff_inspector to check increment2 data now! check diff successfully 4 dm-master alive 3 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" --> (fail) validation update: non-existed subtask dmctl test cmd: "validation update xxxxx" --> (fail) validation update: non-exist source dmctl test cmd: "validation update -s xxxxx test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" --> (fail) validation update: not specify cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-02" --> (fail) validation update: not specify cutover-binlog-pos dmctl test cmd: "validation update test2 -s mysql-replica-01" [Mon May 6 16:56:00 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS optimistic >>>>>> [Mon May 6 16:56:00 CST 2024] <<<<<< start DM-COLUMN_INDEX pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8262 alive the 4-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 303k 0 --:--:-- --:--:-- --:--:-- 362k --> (fail) validation update: invalid cutover-binlog-pos dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos xxx" --> (fail) validation update: specify both cutover-binlog-pos and cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos '(mysql-bin.000001, 2345)' --cutover-binlog-gtid adaa546c-0b85-11ef-838e-422a3ffab8f0:1-999" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... [Mon May 6 16:56:00 CST 2024] <<<<<< finish DM-4219 optimistic >>>>>> --> (success) validation update: on exist source dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid adaa546c-0b85-11ef-838e-422a3ffab8f0:1-999" dmctl test cmd: "validation status test" ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. got=1 expected=1 got=0 expected=0 got=2 expected=2 got=2 expected=2 got=0 expected=0 got=2 expected=2 got=1 expected=1 dmctl test cmd: "validation status test2" got=1 expected=1 got=0 expected=0 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 --> (success) validation stop: stop all tasks dmctl test cmd: "validation stop --all-task" wait process dm-master.test exit... dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=0 got=2 expected=2 got=0 expected=0 got=2 expected=2 dmctl test cmd: "validation status test2" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8262 alive the 5-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 801 100 801 0 0 96459 0 --:--:-- --:--:-- --:--:-- 97k wait process dm-master.test exit... got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 16:56:01 CST 2024] <<<<<< start DM-4220 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8262 alive the 6-th time wait process dm-master.test exit... wait process dm-master.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 kill dm-worker1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8262 alive the 7-th time wait process dm-worker1 exit... dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:16594" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:17605" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:16787" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:17824" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:16787" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:56:04 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --worker --name=worker1" got=1 expected=1 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 7786 0 --:--:-- --:--:-- --:--:-- 8250 check data check diff successfully ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. wait process dm-worker.test exit... dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:17824" dmctl test cmd: "query-status test" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:56:05 CST 2024] <<<<<< finish DM-COLUMN_INDEX pessimistic >>>>>> [Mon May 6 16:56:05 CST 2024] <<<<<< start DM-COLUMN_INDEX optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:04 CST 2024] <<<<<< test case shardddl3 success! >>>>>> start running case: [shardddl3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8263 alive the 2-th time run tidb sql failed 1-th time, retry later alter: add column c int comment '1 2 3๐Ÿ˜Š4'; wait process dm-master.test exit... wait process dm-worker.test exit... check sync diff after set SafeModeInitPhaseSeconds failpoint check diff successfully wait for one of DM-worker offine failed, retry later wait process dm-master.test exit... pt-online-schema-change 3.0.12 wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:56:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait for one of DM-worker offine failed, retry later wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:06 CST 2024] <<<<<< test case http_apis success! >>>>>> start running case: [new_relay] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh... Verbose mode = false wait for rpc addr 127.0.0.1:8263 alive the 3-th time wait process dm-master.test exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive use sync_diff_inspector to check full data dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:16787" wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> check diff successfully alter: add column age int; 1.1.0 DM-worker1 is sharding lock owner and detects it offline [Mon May 6 16:56:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:17824" dmctl test cmd: "binlog revert test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:17824" dmctl test cmd: "binlog revert test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source1.yaml" [Mon May 6 16:56:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8263 alive the 4-th time [Mon May 6 16:56:07 CST 2024] <<<<<< finish DM-COLUMN_INDEX optimistic >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "stop-task test" [Mon May 6 16:56:08 CST 2024] <<<<<< finish DM-4220 pessimistic >>>>>> [Mon May 6 16:56:08 CST 2024] <<<<<< start DM-4220 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source2.yaml" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:56:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8263 alive the 5-th time wait process dm-worker exit... rpc addr 127.0.0.1:8262 is alive alter: add column age int; 1.1.0 wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait for rpc addr 127.0.0.1:8263 alive the 6-th time wait process dm-worker exit... process dm-worker already exit [Mon May 6 16:56:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Mon May 6 16:56:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" alter: add key name (name); 1.1.0 [Mon May 6 16:56:09 CST 2024] <<<<<< start DM-099 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:19218" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:20496" after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 check log contain failed 1-th time, retry later check diff successfully dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:19411" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:20715" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8263 alive the 7-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:56:10 CST 2024] <<<<<< start DM-DML_EXECUTE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:20715" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "check-task /tmp/dm_test/dmctl_command/temp.yaml" check diff successfully run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive start dm-master1 and dm-master2 again [Mon May 6 16:56:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Mon May 6 16:56:11 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Mon May 6 16:56:10 CST 2024] <<<<<< finish test_kill_worker_in_sync >>>>>> 3 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-master.test exit... alter: add key name (name); 1.1.0 check log contain failed 2-th time, retry later wait process dm-master.test exit... wait process dm-worker2 exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" wait process dm-master.test exit... dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:56:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:20715" dmctl test cmd: "binlog revert test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:20715" dmctl test cmd: "binlog revert test" dmctl test cmd: "stop-task test" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" [Mon May 6 16:56:14 CST 2024] <<<<<< finish DM-DML_EXECUTE_ERROR >>>>>> [Mon May 6 16:56:14 CST 2024] <<<<<< finish DM-4220 optimistic >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "stop-task test" check log contain failed 3-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... [Mon May 6 16:56:14 CST 2024] <<<<<< finish DM-099 pessimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "list-member --master" got=5 expected=5 [Mon May 6 16:56:15 CST 2024] <<<<<< start test_evict_leader >>>>>> wait process dm-master.test exit... wait process dm-worker exit... wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:56:15 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... leader is master4 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 11781 0 --:--:-- --:--:-- --:--:-- 16500 [Mon May 6 16:56:15 CST 2024] <<<<<< start DM-100 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 new leader is master5 wait process dm-master.test exit... wait process dm-worker.test exit... leader is master5 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 13959 0 --:--:-- --:--:-- --:--:-- 16500 new leader is master1 wait process dm-master.test exit... wait process dm-worker exit... process dm-worker already exit [Mon May 6 16:56:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Mon May 6 16:56:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 4-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:16 CST 2024] <<<<<< test case dmctl_command success! >>>>>> wait process dm-master.test exit... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) leader is master1 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 16344 0 --:--:-- --:--:-- --:--:-- 33000 [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // withCredentials new leader is master2 [Pipeline] } wait process dm-master.test exit... wait process dm-worker.test exit... [Pipeline] // timeout [Pipeline] } dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:21842" leader is master2 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 16491 0 --:--:-- --:--:-- --:--:-- 33000 [Pipeline] // stage [Pipeline] } wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:56:17 CST 2024] <<<<<< start DM-KEY_NOT_FOUND >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] // container dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:23387" new leader is master3 [Pipeline] } wait process dm-master.test exit... [Pipeline] // withEnv [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" leader is master3 dmctl test cmd: "operate-leader evict" [Pipeline] // node wait process dm-worker.test exit... [Pipeline] } wait process dm-master.test exit... dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:18 CST 2024] <<<<<< test case case_sensitive success! >>>>>> % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 15270 0 --:--:-- --:--:-- --:--:-- 16500 cancel evict leader on master1, and master1 will be the leader dmctl test cmd: "operate-leader cancel-evict" cancel evict leader twice, and test cancel evict leader from http interface % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 18922 0 --:--:-- --:--:-- --:--:-- 33000 [Pipeline] // podTemplate [Pipeline] } got=3 expected=3 dmctl test cmd: "stop-task test" start running case: [checkpoint_transaction] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> [Pipeline] // withEnv leader is master1 cancel evict leader on all masters cancel master port 8361 dmctl test cmd: "operate-leader cancel-evict" cancel master port 8461 dmctl test cmd: "operate-leader cancel-evict" dmctl test cmd: "query-status test" wait process dm-master.test exit... [Pipeline] } check log contain failed 5-th time, retry later alter: add column age int; 1.1.0 [Mon May 6 16:56:18 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Mon May 6 16:56:18 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] // stage wait for rpc addr 127.0.0.1:8261 alive the 1-th time cancel master port 8561 dmctl test cmd: "operate-leader cancel-evict" cancel master port 8661 dmctl test cmd: "operate-leader cancel-evict" got=2 expected=2 check diff successfully [Pipeline] } [Mon May 6 16:56:18 CST 2024] <<<<<< finish test_evict_leader >>>>>> [Mon May 6 16:56:18 CST 2024] <<<<<< start test_list_member_command >>>>>> dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:18 CST 2024] <<<<<< test case safe_mode success! >>>>>> start running case: [sequence_safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:19 CST 2024] <<<<<< test case ha_cases_2 success! >>>>>> start running case: [ha_cases2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh... Verbose mode = false wait process dm-master.test exit... got=2 expected=2 check diff successfully check diff successfully dmctl test cmd: "stop-task test" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:19 CST 2024] <<<<<< start test_pause_task >>>>>> [Mon May 6 16:56:19 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 16:56:19 CST 2024] <<<<<< finish DM-KEY_NOT_FOUND >>>>>> current leader is master1 dmctl test cmd: "list-member --master" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-worker1 exit... got=5 expected=5 kill leader master1 got=2 expected=2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_leak passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:23811" alter: add column age int; 1.1.0 wait process dm-worker exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:25545" wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 16:56:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process master1 exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Mon May 6 16:56:21 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> [Mon May 6 16:56:21 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker exit... process dm-worker already exit [Mon May 6 16:56:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Mon May 6 16:56:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... import prepare data start DM worker and master [Mon May 6 16:56:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>> [Mon May 6 16:56:21 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>> [Mon May 6 16:56:21 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/source1.yaml" rpc addr 127.0.0.1:8262 is alive wait process master1 exit... process master1 already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-task.yaml " rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:56:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:56:22 CST 2024] <<<<<< finish DM-100 optimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... current leader is master2 dmctl test cmd: "list-member --master" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:56:22 CST 2024] <<<<<< start DM-001 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" alter: add key name (name); 1.1.0 got=4 expected=4 kill leader master2 dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:56:23 CST 2024] <<<<<< start DM-101 optimistic >>>>>> check diff successfully rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source2.yaml" wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:26038" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:27703" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process master2 exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Mon May 6 16:56:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-worker.test exit... [Mon May 6 16:56:24 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Mon May 6 16:56:24 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:56:24 CST 2024] <<<<<< finish DM-001 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-task.yaml --remove-meta" alter: add key name (name); 1.1.0 dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process master2 exit... process master2 already exit wait process dm-master.test exit... [Mon May 6 16:56:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time kill dm-master1 wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 16:56:25 CST 2024] <<<<<< start DM-002 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait for port 8262 offline the 1-th time wait process dm-master.test exit... wait process dm-worker2 exit... [Mon May 6 16:56:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:26 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:28265" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:29861" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait for port 8262 offline the 2-th time got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker2 exit... process dm-worker2 already exit [Mon May 6 16:56:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 16:56:27 CST 2024] <<<<<< finish DM-002 >>>>>> alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:56:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "stop-task test" current leader is master5 dmctl test cmd: "list-member --master" [Mon May 6 16:56:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:56:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... [Mon May 6 16:56:28 CST 2024] <<<<<< finish DM-101 optimistic >>>>>> [Mon May 6 16:56:28 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> got=5 expected=5 [Mon May 6 16:56:28 CST 2024] <<<<<< start DM-003 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8264 alive the 1-th time current leader is master5 kill master1 rpc addr 127.0.0.1:8263 is alive [Mon May 6 16:56:28 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "stop-task test" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... start check_worker_ungraceful_stop_with_retry times: 0 [Mon May 6 16:56:29 CST 2024] <<<<<< start DM-102 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 16:56:29 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8264 is alive [Mon May 6 16:56:29 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait process dm-master1 exit... rpc addr 127.0.0.1:8264 is alive got=1 expected=1 check diff failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... start check_worker_ungraceful_stop_with_retry times: 1 dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1 worker2" dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:56:30 CST 2024] <<<<<< start DM-4201 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:18262 is alive [Mon May 6 16:56:30 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time wait process dm-master1 exit... process dm-master1 already exit alter: add column age int; check sync diff after reset failpoint start check_worker_ungraceful_stop_with_retry times: 2 check_worker_ungraceful_stop_with_retry success after retry: 2 after ungraceful exit data in source count: 31 data in tidb count: 17 ungraceful stop test success [Mon May 6 16:56:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time pt-online-schema-change 3.0.12 check diff successfully alter: add column age int; wait process dm-master.test exit... dmctl test cmd: "query-status test" wait process dm-master.test exit... pt-online-schema-change 3.0.12 rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml " got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=3 expected=3 dmctl test cmd: "transfer-source mysql-replica-01 worker1" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=3 expected=3 got=1 expected=1 got=1 expected=1 alter: add key name (name); dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:30234" dmctl test cmd: "query-status test" check diff failed 2-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... pt-online-schema-change 3.0.12 got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process dm-master.test exit... alter: add key name (name); dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb`" pt-online-schema-change 3.0.12 wait for rpc addr 127.0.0.1:8261 alive the 2-th time alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; wait process dm-worker.test exit... process dm-worker.test already exit restart DM-worker after set SequenceShardSyncedExecutionExit failpoint [Mon May 6 16:56:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Mon May 6 16:56:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader got=1 expected=1 dmctl test cmd: "stop-task test" got=2 expected=2 pt-online-schema-change 3.0.12 wait process dm-master.test exit... check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase [Mon May 6 16:56:33 CST 2024] <<<<<< finish DM-102 >>>>>> check diff successfully check diff failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; dmctl test cmd: "stop-task test" [Mon May 6 16:56:34 CST 2024] <<<<<< finish DM-4201 >>>>>> pt-online-schema-change 3.0.12 check diff failed 3-th time, retry later alter: add column age int; wait process dm-master.test exit... pt-online-schema-change 3.0.12 [Mon May 6 16:56:34 CST 2024] <<<<<< start DM-103 pessimistic >>>>>> rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... alter: add column age int; dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive pt-online-schema-change 3.0.12 alter: add key name (name); wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit pt-online-schema-change 3.0.12 wait process dm-master.test exit... alter: add key name (name); wait process dm-master.test exit... pt-online-schema-change 3.0.12 check diff failed at last check diff successfully dmctl test cmd: "stop-task test" check diff successfully kill dm-worker1 dmctl test cmd: "query-status test" got=2 expected=2 alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; [Mon May 6 16:56:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later wait process dm-master.test exit... wait for one of DM-worker offine failed, retry later pt-online-schema-change 3.0.12 [Mon May 6 16:56:36 CST 2024] <<<<<< finish DM-003 pessimistic >>>>>> wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member --master" pt-online-schema-change 3.0.12 wait process dm-worker1 exit... wait process dm-master.test exit... process dm-master.test already exit got=5 expected=5 current leader is master5 kill master2 use sync_diff_inspector to check increment data check diff successfully start dm-worker3 and kill dm-worker2 wait process dm-master.test exit... wait for one of DM-worker offine failed, retry later wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=2 expected=2 wait process dm-worker.test exit... [Mon May 6 16:56:37 CST 2024] <<<<<< start DM-004 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-optimistic.yaml --remove-meta" check diff successfully [Mon May 6 16:56:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' [Mon May 6 16:56:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master-no-tls.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... wait process dm-master.test exit... DM-worker1 is sharding lock owner and detects it offline [Mon May 6 16:56:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master2 exit... use sync_diff_inspector to check increment data check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:56:38 CST 2024] <<<<<< finish DM-103 pessimistic >>>>>> [Mon May 6 16:56:38 CST 2024] <<<<<< start DM-103 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff successfully check diff successfully [Mon May 6 16:56:39 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "query-status -s mysql-replica-01" got=3 expected=3 got=1 expected=1 got=1 expected=1 check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit [Mon May 6 16:56:39 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker3.toml >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive wait process dm-master2 exit... process dm-master2 already exit wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:39 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /tmp/dm_test/tls/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:40 CST 2024] <<<<<< test case handle_error_2 success! >>>>>> start running case: [handle_error_3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" check diff failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8264 is alive wait and check task running dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "purge-relay --filename dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9-bin.000005 -s mysql-replica-01" kill dm-worker1 rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-no-tls.yaml" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later wait process dm-master.test exit... wait process dm-worker1 exit... got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=2 expected=2 start dumping SQLs into source pause tasks test dmctl test cmd: "pause-task test" dmctl test cmd: "pause-task test" got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-no-tls.yaml --remove-meta=true" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source1.yaml" pause tasks test2 dmctl test cmd: "pause-task test2" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:56:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "pause-task test2" wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test2" wait process dm-master.test exit... [Mon May 6 16:56:43 CST 2024] <<<<<< finish DM-103 optimistic >>>>>> [Mon May 6 16:56:43 CST 2024] <<<<<< finish DM-004 optimistic >>>>>> check diff failed 4-th time, retry later use sync_diff_inspector to check increment2 data now! check diff successfully 1 dm-master alive got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check data check diff successfully ============================== test_source_and_target_with_empty_tlsconfig success ================================== rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source2.yaml" 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... [Mon May 6 16:56:44 CST 2024] <<<<<< start DM-104 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker2 exit... resume tasks test dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "resume-task test" [Mon May 6 16:56:44 CST 2024] <<<<<< start DM-005 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "query-status test" resume tasks test2 dmctl test cmd: "resume-task test2" wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-master.test exit... [Mon May 6 16:56:44 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 check diff failed 1-th time, retry later got=3 expected=3 dmctl test cmd: "resume-task test2" check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 16:56:44 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time check diff failed 5-th time, retry later got=3 expected=3 dmctl test cmd: "query-status test2" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit run tidb sql failed 1-th time, retry later got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Mon May 6 16:56:46 CST 2024] <<<<<< finish DM-005 >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8361 is alive dmctl test cmd: "list-member --master" got=5 expected=5 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" current leader is master5 kill master3 wait process dm-worker.test exit... got=2 expected=2 wait process dm-master.test exit... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:1559 alter table handle_error.tb1 add column d varchar(20);" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:1735 alter table handle_error.tb1 add column d varchar(20);" wait process dm-master3 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "config export -p /tmp/configs" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... check diff failed 6-th time, retry later [Mon May 6 16:56:47 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Mon May 6 16:56:47 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... [Mon May 6 16:56:47 CST 2024] <<<<<< finish DM-104 optimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master3 exit... process dm-master3 already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:48 CST 2024] <<<<<< test case online_ddl success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Pipeline] // dir [Mon May 6 16:56:48 CST 2024] <<<<<< start DM-105 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit [Pipeline] // withCredentials wait process dm-worker.test exit... [Pipeline] } [Pipeline] // timeout dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] } wait process dm-master.test exit... process dm-master.test already exit [Pipeline] // stage [Pipeline] } check diff successfully pause task and check status dmctl test cmd: "pause-task test" [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:3856 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:4247 alter table handle_error.tb1 add column d varchar(20);" [Pipeline] // node [Pipeline] } wait process dm-worker.test exit... [Pipeline] // podTemplate wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:49 CST 2024] <<<<<< test case sequence_safe_mode success! >>>>>> [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" [Pipeline] // withEnv [Pipeline] } [Pipeline] } wait process dm-worker.test exit... Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 [Pipeline] // stage [Pipeline] // cache [Pipeline] } [Pipeline] } run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test" [Pipeline] // dir wait process dm-master.test exit... [Pipeline] } wait process dm-worker.test exit... [Pipeline] // withCredentials [Mon May 6 16:56:50 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> [Mon May 6 16:56:50 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" [Pipeline] } wait process dm-worker.test exit... [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:51 CST 2024] <<<<<< test case tls success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Pipeline] // withEnv wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:52 CST 2024] <<<<<< test case shardddl1 success! >>>>>> start running case: [shardddl1_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] } [Pipeline] // node [Pipeline] } dmctl test cmd: "stop-task test" [Pipeline] // podTemplate got=2 expected=2 [Pipeline] } wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:56:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:6411 alter table handle_error.tb1 add column d varchar(20);" [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:6759 alter table handle_error.tb1 add column d varchar(20);" wait process dm-master.test exit... [Mon May 6 16:56:52 CST 2024] <<<<<< finish DM-105 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:56:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Mon May 6 16:56:53 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time [Mon May 6 16:56:53 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Mon May 6 16:56:53 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> wait process dm-master.test exit... [Mon May 6 16:56:53 CST 2024] <<<<<< start DM-106 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8461 is alive dmctl test cmd: "list-member --master" got=5 expected=5 rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:56:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... current leader is master5 kill master4 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Mon May 6 16:56:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "config import -p /tmp/configs" wait process dm-master.test exit... wait process dm-master4 exit... dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:8966 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:9271 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source2.yaml" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Mon May 6 16:56:56 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> wait process dm-master4 exit... process dm-master4 already exit wait process dm-master.test exit... [Mon May 6 16:56:57 CST 2024] <<<<<< start DM-SAME_DDL_TWICE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" [Mon May 6 16:56:57 CST 2024] <<<<<< finish DM-106 optimistic >>>>>> [Mon May 6 16:56:57 CST 2024] <<<<<< start DM-4210 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "operate-source show" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 check diff successfully >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_operations passed 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 16:56:58 CST 2024] <<<<<< start DM-107 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-master.test exit... dmctl test cmd: "binlog replace test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:11239 alter table handle_error.tb1 add column d int;alter table handle_error.tb1 add unique(d);" dmctl test cmd: "query-status test" check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column e int unique;" dmctl test cmd: "query-status test" wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:57:02 CST 2024] <<<<<< finish DM-SAME_DDL_TWICE pessimistic >>>>>> [Mon May 6 16:57:02 CST 2024] <<<<<< start DM-006 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 16:57:02 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time [Mon May 6 16:57:02 CST 2024] <<<<<< finish DM-4210 >>>>>> wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:02 CST 2024] <<<<<< test case ha success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Pipeline] } rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "list-member --master" got=5 expected=5 [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Mon May 6 16:57:03 CST 2024] <<<<<< start DM-4193 pessimistic >>>>>> [Mon May 6 16:57:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // withEnv current leader is master5 dmctl test cmd: "list-member --worker --name=worker1,worker2" [Pipeline] } dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" [Pipeline] // node check diff failed 1-th time, retry later [Pipeline] } got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source1.yaml" dmctl test cmd: "list-member --worker" [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source2.yaml" [Pipeline] // stage [Pipeline] } dmctl test cmd: "list-member" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:57:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" graceful pause test success start check pause diff check diff successfully resume task and check status dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" kill dm-worker1 [Mon May 6 16:57:05 CST 2024] <<<<<< finish DM-107 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:13027" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:12152" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" [Mon May 6 16:57:06 CST 2024] <<<<<< start DM-108 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:13027" dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:12152" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:13413" wait process dm-worker1 exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:57:06 CST 2024] <<<<<< finish DM-006 >>>>>> dmctl test cmd: "binlog revert test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:13413" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:13413" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:12590" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 kill dm-worker1 wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 16:57:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 [Mon May 6 16:57:07 CST 2024] <<<<<< start DM-007 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "list-member --name=worker1,worker2" got=2 expected=2 kill worker1 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Mon May 6 16:57:07 CST 2024] <<<<<< finish DM-4193 pessimistic >>>>>> [Mon May 6 16:57:07 CST 2024] <<<<<< start DM-4193 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker1 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 stop task dmctl test cmd: "stop-task test" wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 16:57:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:15189" wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --name=worker1,worker2" got=1 expected=1 got=1 expected=1 kill worker2 check diff failed 2-th time, retry later dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:14529" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:15189" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:14529" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:15575" got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_upstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog revert test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:15575" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:15575" wait process dm-worker2 exit... dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:14967" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:57:11 CST 2024] <<<<<< finish DM-007 >>>>>> got=3 expected=3 run tidb sql failed 1-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member" got=2 expected=2 [Mon May 6 16:57:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:57:12 CST 2024] <<<<<< start DM-008 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check diff failed 3-th time, retry later rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:57:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" check diff failed at last dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "config master master1" [Mon May 6 16:57:14 CST 2024] <<<<<< finish DM-108 optimistic >>>>>> got=1 expected=1 dmctl test cmd: "config master master2" got=1 expected=1 [Mon May 6 16:57:14 CST 2024] <<<<<< finish test_list_member_command >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 16:57:15 CST 2024] <<<<<< start DM-109 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" [Mon May 6 16:57:15 CST 2024] <<<<<< finish DM-4193 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:57:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-task.yaml --remove-meta" [Mon May 6 16:57:16 CST 2024] <<<<<< finish DM-008 >>>>>> [Mon May 6 16:57:16 CST 2024] <<<<<< start DM-4230 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source1.yaml" [Mon May 6 16:57:17 CST 2024] <<<<<< start DM-009 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file kill dm-master1 and kill dm-master2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column d int unique;" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog revert test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 [Mon May 6 16:57:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 1-th time, retry later [Mon May 6 16:57:18 CST 2024] <<<<<< finish DM-4230 >>>>>> wait process dm-master1 exit... check diff failed 2-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source2.yaml" [Mon May 6 16:57:19 CST 2024] <<<<<< start DM-4177 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master1 exit... process dm-master1 already exit check diff successfully dmctl test cmd: "stop-task test" wait process dm-master2 exit... dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" [Mon May 6 16:57:21 CST 2024] <<<<<< finish DM-009 >>>>>> check diff failed 3-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip wrong-task" dmctl test cmd: "binlog skip" dmctl test cmd: "binlog skip test -b dm-it-49eecccb-6e1b-47ce-bb47-1b1bf7ecddd1-kkt36-jq5p0-bin|000001.000001:19397 -s wrong-source" wait process dm-master2 exit... process dm-master2 already exit wait and check task running dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=2 got=1 expected=1 [Mon May 6 16:57:22 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> dmctl test cmd: "binlog skip test -b mysql-bin|1111 -s wrong-source" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" HTTP 127.0.0.1:8461/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8264 alive the 1-th time [Mon May 6 16:57:22 CST 2024] <<<<<< start DM-010 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int;"" got=3 expected=3 dmctl test cmd: "binlog skip test -b mysql-bin.000000:00000" dmctl test cmd: "binlog aaa test" dmctl test cmd: "stop-task test" [Mon May 6 16:57:22 CST 2024] <<<<<< finish DM-4177 >>>>>> dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3" dmctl test cmd: "binlog revert test" check diff failed at last dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 [Mon May 6 16:57:23 CST 2024] <<<<<< finish DM-109 optimistic >>>>>> check diff failed 1-th time, retry later dmctl test cmd: "query-status -s mysql-replica-02" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -n worker3" [Mon May 6 16:57:24 CST 2024] <<<<<< start DM-4231 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" use sync_diff_inspector to check data now! check diff successfully dmctl test cmd: "offline-member --master --name master1" dmctl test cmd: "offline-member --master --name master2" kill dm-master3 [Mon May 6 16:57:24 CST 2024] <<<<<< start DM-110 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int; alter table handle_error.tb1 add column d int unique;"" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-master3 exit... got=2 expected=2 dmctl test cmd: "query-status test" [Mon May 6 16:57:26 CST 2024] <<<<<< finish DM-010 >>>>>> graceful stop test success start check stop diff check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" got=2 expected=2 got=1 expected=1 check diff failed 1-th time, retry later [Mon May 6 16:57:26 CST 2024] <<<<<< finish DM-4231 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit wait process dm-worker.test exit... wait process dm-master3 exit... process dm-master3 already exit [Mon May 6 16:57:27 CST 2024] <<<<<< start DM-011 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:57:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:28 CST 2024] <<<<<< test case checkpoint_transaction success! >>>>>> start running case: [check_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "stop-task test" wait process dm-master.test exit... check diff failed 2-th time, retry later [Mon May 6 16:57:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" [Mon May 6 16:57:28 CST 2024] <<<<<< finish DM-011 >>>>>> got=3 expected=3 [Mon May 6 16:57:29 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 16:57:29 CST 2024] <<<<<< start DM-012 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8461 is alive wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:57:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:57:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> [Mon May 6 16:57:31 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 16:57:31 CST 2024] <<<<<< finish DM-012 >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:32 CST 2024] <<<<<< test case handle_error_3 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-noshard.yaml" [Pipeline] // dir rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8264 is alive [Mon May 6 16:57:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 16:57:32 CST 2024] <<<<<< start DM-013 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-sharding.yaml" --> start test_privileges_can_migrate... [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml --remove-meta" [Pipeline] // node [Pipeline] } check diff failed at last dmctl test cmd: "stop-task test" [Mon May 6 16:57:33 CST 2024] <<<<<< finish DM-110 optimistic >>>>>> [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-02" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member --worker" >>>>>>>>>>>>>>>>>>>>>>>>>>test test_restart_relay_status passed 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:57:34 CST 2024] <<<<<< start DM-111 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:57:34 CST 2024] <<<<<< finish DM-013 >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:57:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 16:57:35 CST 2024] <<<<<< start DM-014 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 pass test_privileges_can_migrate --> start test_privilege_precheck... dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" pass test_privilege_precheck 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "pause-task test" got=3 expected=3 got=1 expected=1 got=1 expected=1 4 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... check diff failed 2-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "stop-task test" [Mon May 6 16:57:39 CST 2024] <<<<<< finish DM-014 >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:57:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later [Mon May 6 16:57:40 CST 2024] <<<<<< start DM-015 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start running case: [dm_syncer] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/run.sh... Verbose mode = false wait process dm-master.test exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" [Mon May 6 16:57:42 CST 2024] <<<<<< finish DM-111 optimistic >>>>>> wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:57:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:57:43 CST 2024] <<<<<< start DM-112 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:57:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... kill dm-worker1 [Mon May 6 16:57:44 CST 2024] <<<<<< finish DM-015 >>>>>> dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-worker1 exit... got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dm_syncer/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dm_syncer/source2.yaml" [Mon May 6 16:57:45 CST 2024] <<<<<< start DM-016 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:45 CST 2024] <<<<<< test case ha_master success! >>>>>> [Pipeline] } wait process dm-worker1 exit... process dm-worker1 already exit Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } dmctl test cmd: "start-task /tmp/dm_test/dm_syncer/dm-task.yaml --remove-meta" [Pipeline] // stage dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Pipeline] } wait process tidb-server exit... process tidb-server already exit [Mon May 6 16:57:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:57:46 CST 2024] <<<<<< finish DM-016 >>>>>> check diff failed 2-th time, retry later check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 16:57:47 CST 2024] <<<<<< start DM-017 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=1 expected=1 check log contain failed 1-th time, retry later check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_downstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/bin/dm-syncer.test --log-file=/tmp/dm_test/dm_syncer/syncer1/log/dm-syncer.log --config=/tmp/dm_test/dm_syncer/dm-syncer-1.toml >> /tmp/dm_test/dm_syncer/syncer1/log/stdout.log 2>&1 & /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/bin/dm-syncer.test --log-file=/tmp/dm_test/dm_syncer/syncer2/log/dm-syncer.log --config=/tmp/dm_test/dm_syncer/dm-syncer-2.toml --meta=/tmp/dm_test/dm_syncer/old_meta_file --syncer-config-format >> /tmp/dm_test/dm_syncer/syncer2/log/stdout.log 2>&1 & dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Mon May 6 16:57:51 CST 2024] <<<<<< finish DM-017 >>>>>> check diff failed at last dmctl test cmd: "stop-task test" [Mon May 6 16:57:51 CST 2024] <<<<<< finish DM-112 optimistic >>>>>> wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:57:52 CST 2024] <<<<<< start DM-113 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 16:57:52 CST 2024] <<<<<< start DM-018 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:57:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 16:57:56 CST 2024] <<<<<< finish DM-018 >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:57:56 CST 2024] <<<<<< finish DM-113 pessimistic >>>>>> [Mon May 6 16:57:56 CST 2024] <<<<<< start DM-113 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:57:57 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:57:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 16:57:59 CST 2024] <<<<<< finish DM-019 >>>>>> [Mon May 6 16:57:59 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 [Mon May 6 16:57:59 CST 2024] <<<<<< finish DM-113 optimistic >>>>>> dmctl test cmd: "query-status -s mysql-replica-01" got=0 expected=1 command: query-status -s mysql-replica-01 "relayCatchUpMaster": true count: 0 != expected: 1, failed the 0-th time, will retry again check diff successfully check diff successfully check diff successfully 1 dm-master alive 0 dm-worker alive 2 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 16:58:00 CST 2024] <<<<<< finish DM-019 >>>>>> [Mon May 6 16:58:00 CST 2024] <<<<<< start DM-114 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... [Mon May 6 16:58:01 CST 2024] <<<<<< start DM-020 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_kill_dump_connection passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:02 CST 2024] <<<<<< test case dm_syncer success! >>>>>> start running case: [downstream_diff_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" wait process dm-master.test exit... got=1 expected=1 run tidb sql failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:04 CST 2024] <<<<<< finish DM-114 pessimistic >>>>>> [Mon May 6 16:58:04 CST 2024] <<<<<< start DM-114 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 16:58:05 CST 2024] <<<<<< finish DM-020 >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source1.yaml" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:06 CST 2024] <<<<<< start DM-021 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" [Mon May 6 16:58:06 CST 2024] <<<<<< finish DM-114 optimistic >>>>>> [Mon May 6 16:58:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 16:58:07 CST 2024] <<<<<< start DM-115 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source2.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:08 CST 2024] <<<<<< test case new_relay success! >>>>>> start running case: [all_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:08 CST 2024] <<<<<< start test_expression_filter >>>>>> [Mon May 6 16:58:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:58:10 CST 2024] <<<<<< finish DM-021 >>>>>> run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:58:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 16:58:11 CST 2024] <<<<<< start DM-022 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:11 CST 2024] <<<<<< finish DM-115 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 16:58:12 CST 2024] <<<<<< start DM-116 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later check diff successfully [Mon May 6 16:58:15 CST 2024] <<<<<< finish DM-022 >>>>>> check diff failed 1-th time, retry later wait process dm-worker.test exit... [Mon May 6 16:58:16 CST 2024] <<<<<< start DM-023 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 16:58:17 CST 2024] <<<<<< finish DM-116 pessimistic >>>>>> [Mon May 6 16:58:17 CST 2024] <<<<<< start DM-116 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" check diff failed 2-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:17 CST 2024] <<<<<< test case downstream_diff_index success! >>>>>> start running case: [downstream_more_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 16:58:17 CST 2024] <<<<<< finish DM-023 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:18 CST 2024] <<<<<< start DM-026 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" [Mon May 6 16:58:19 CST 2024] <<<<<< finish DM-116 optimistic >>>>>> check diff failed 3-th time, retry later [Mon May 6 16:58:20 CST 2024] <<<<<< start DM-117 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/downstream_more_column/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:20 CST 2024] <<<<<< finish DM-026 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff failed at last dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive use sync_diff_inspector to check increment data check diff successfully check diff successfully [Mon May 6 16:58:10 CST 2024] <<<<<< finish test_pause_task >>>>>> [Mon May 6 16:58:10 CST 2024] <<<<<< start test_multi_task_reduce_and_restart_worker >>>>>> [Mon May 6 16:58:10 CST 2024] <<<<<< start test_multi_task_running >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master [Mon May 6 16:58:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>> [Mon May 6 16:58:21 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>> [Mon May 6 16:58:21 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 16:58:21 CST 2024] <<<<<< start DM-027 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl3_1/schema.sql -s mysql-replica-01" dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" run tidb sql failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Mon May 6 16:58:22 CST 2024] <<<<<< finish DM-117 optimistic >>>>>> rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Mon May 6 16:58:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 16:58:23 CST 2024] <<<<<< start DM-118 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 16:58:23 CST 2024] <<<<<< finish DM-027 >>>>>> wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/downstream_more_column/dm-task-incremental.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 16:58:24 CST 2024] <<<<<< start DM-028 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check diff failed 1-th time, retry later [Mon May 6 16:58:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d downstream_more_column1 -t t1 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/data/schema.sql" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=2 expected=2 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Mon May 6 16:58:26 CST 2024] <<<<<< finish DM-028 >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:26 CST 2024] <<<<<< finish test_expression_filter >>>>>> [Mon May 6 16:58:26 CST 2024] <<<<<< start test_json_expression >>>>>> wait process dm-master.test exit... [Mon May 6 16:58:27 CST 2024] <<<<<< start DM-030 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 16:58:27 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" [Mon May 6 16:58:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8264 is alive [Mon May 6 16:58:28 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time got=2 expected=2 check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:29 CST 2024] <<<<<< finish DM-118 pessimistic >>>>>> [Mon May 6 16:58:29 CST 2024] <<<<<< start DM-118 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:18262 is alive [Mon May 6 16:58:29 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:58:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml " wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:31 CST 2024] <<<<<< test case downstream_more_column success! >>>>>> start running case: [drop_column_with_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:31 CST 2024] <<<<<< finish DM-030 pessimistic >>>>>> [Mon May 6 16:58:31 CST 2024] <<<<<< start DM-030 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:58:31 CST 2024] <<<<<< finish DM-118 optimistic >>>>>> [Mon May 6 16:58:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/drop_column_with_index/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:34 CST 2024] <<<<<< finish DM-030 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" [Mon May 6 16:58:35 CST 2024] <<<<<< start DM-031 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" check diff failed 1-th time, retry later got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "stop-task test" [Mon May 6 16:58:37 CST 2024] <<<<<< finish DM-031 pessimistic >>>>>> [Mon May 6 16:58:37 CST 2024] <<<<<< start DM-031 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:37 CST 2024] <<<<<< test case shardddl3_1 success! >>>>>> start running case: [shardddl4] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... [Mon May 6 16:58:39 CST 2024] <<<<<< finish DM-031 optimistic >>>>>> use sync_diff_inspector to check increment data check diff successfully check diff successfully [Mon May 6 16:58:38 CST 2024] <<<<<< finish test_multi_task_running >>>>>> start dumping SQLs into source "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) restart unuse worker3 try to kill worker port 8264 wait process dm-worker3 exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source1.yaml" [Mon May 6 16:58:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 16:58:40 CST 2024] <<<<<< start DM-032 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" process dm-master.test already exit wait process dm-worker3 exit... process dm-worker3 already exit dmctl test cmd: "list-member --worker --name=worker3" got=1 expected=1 start dm-worker3 [Mon May 6 16:58:40 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source2.yaml" wait process dm-worker.test exit... dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again check diff failed 1-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8264 is alive restart unuse worker4 try to kill worker port 18262 [Mon May 6 16:58:42 CST 2024] <<<<<< start DM-119 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker4 exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:43 CST 2024] <<<<<< finish test_json_expression >>>>>> [Mon May 6 16:58:43 CST 2024] <<<<<< start test_fail_job_between_event >>>>>> [Mon May 6 16:58:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker4 exit... process dm-worker4 already exit dmctl test cmd: "list-member --worker --name=worker4" check diff failed 2-th time, retry later got=1 expected=1 start dm-worker4 [Mon May 6 16:58:44 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:18262 is alive restart unuse worker5 try to kill worker port 18263 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Mon May 6 16:58:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:46 CST 2024] <<<<<< finish DM-032 optimistic >>>>>> check diff failed 2-th time, retry later wait process dm-worker5 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-worker.test exit... [Mon May 6 16:58:47 CST 2024] <<<<<< start DM-033 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker5 exit... process dm-worker5 already exit dmctl test cmd: "list-member --worker --name=worker5" got=1 expected=1 start dm-worker5 [Mon May 6 16:58:47 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task.yaml --remove-meta" check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:48 CST 2024] <<<<<< test case drop_column_with_index success! >>>>>> start running case: [duplicate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:18263 is alive try to kill worker port 8262 dmctl test cmd: "query-status test" got=3 expected=3 check diff failed 1-th time, retry later wait process dm-worker1 exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 4-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit worker1 was killed dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:58:51 CST 2024] <<<<<< finish DM-033 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" check diff failed 5-th time, retry later check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 16:58:52 CST 2024] <<<<<< start DM-034 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 16:58:54 CST 2024] <<<<<< finish DM-034 pessimistic >>>>>> [Mon May 6 16:58:54 CST 2024] <<<<<< start DM-034 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 2-th time, retry later check diff failed 6-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 16:58:56 CST 2024] <<<<<< finish DM-034 optimistic >>>>>> check log contain failed 3-th time, retry later check diff failed 7-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:58:57 CST 2024] <<<<<< finish test_fail_job_between_event >>>>>> [Mon May 6 16:58:57 CST 2024] <<<<<< start test_session_config >>>>>> [Mon May 6 16:58:57 CST 2024] <<<<<< start DM-035 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 16:58:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:58:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> got=2 expected=2 run tidb sql failed 1-th time, retry later wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 4-th time, retry later check diff failed 8-th time, retry later rpc addr 127.0.0.1:8262 is alive [Mon May 6 16:58:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" check log contain failed 5-th time, retry later check diff failed 9-th time, retry later dmctl test cmd: "stop-task test" [Mon May 6 16:59:00 CST 2024] <<<<<< finish DM-035 optimistic >>>>>> dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" check log contain failed 6-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" check diff failed 10-th time, retry later wait process dm-master.test exit... check diff successfully dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed at last dmctl test cmd: "stop-task test" [Mon May 6 16:59:05 CST 2024] <<<<<< finish DM-119 pessimistic >>>>>> [Mon May 6 16:59:05 CST 2024] <<<<<< start DM-119 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit check log contain failed 7-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 8-th time, retry later wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:59:07 CST 2024] <<<<<< test case shardddl1_1 success! >>>>>> start running case: [shardddl2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:59:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... check log contain failed 9-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:59:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 16:59:10 CST 2024] <<<<<< finish test_session_config >>>>>> [Mon May 6 16:59:10 CST 2024] <<<<<< start test_query_timeout >>>>>> [Mon May 6 16:59:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source1.yaml" [Mon May 6 16:59:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 10-th time, retry later check diff failed 3-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 16:59:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source2.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" check log contain failed 11-th time, retry later [Mon May 6 16:59:12 CST 2024] <<<<<< start DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff failed 4-th time, retry later [Mon May 6 16:59:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker 1 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" check log contain failed 12-th time, retry later wait process dm-worker1 exit... check diff failed 5-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 16:59:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 13-th time, retry later rpc addr 127.0.0.1:8262 is alive check log contain failed 1-th time, retry later check diff failed 6-th time, retry later check log contain failed 14-th time, retry later check diff failed 7-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 restart dm-master wait process dm-master exit... check log contain failed 15-th time, retry later wait process dm-master exit... process dm-master already exit check diff failed 8-th time, retry later check log contain failed 16-th time, retry later [Mon May 6 16:59:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 9-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 17-th time, retry later check diff failed 10-th time, retry later rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 restart dm-worker 1 check log contain failed 18-th time, retry later wait process dm-worker1 exit... check diff failed at last dmctl test cmd: "stop-task test" wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 16:59:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 16:59:28 CST 2024] <<<<<< finish DM-119 optimistic >>>>>> check log contain failed 19-th time, retry later [Mon May 6 16:59:29 CST 2024] <<<<<< start DM-120 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=3 expected=3 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 20-th time, retry later check diff failed 1-th time, retry later check diff failed 2-th time, retry later check log contain failed 21-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:59:33 CST 2024] <<<<<< finish DM-120 pessimistic >>>>>> [Mon May 6 16:59:33 CST 2024] <<<<<< start DM-120 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:59:33 CST 2024] <<<<<< finish DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> [Mon May 6 16:59:33 CST 2024] <<<<<< start DM-INIT_SCHEMA optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 22-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later check log contain failed 23-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:59:37 CST 2024] <<<<<< finish DM-120 optimistic >>>>>> restart dm-master wait process dm-master exit... [Mon May 6 16:59:38 CST 2024] <<<<<< start DM-121 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 24-th time, retry later wait process dm-master exit... process dm-master already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 25-th time, retry later [Mon May 6 16:59:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 2-th time, retry later check log contain failed 26-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" check diff failed 3-th time, retry later got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 16:59:44 CST 2024] <<<<<< finish DM-INIT_SCHEMA optimistic >>>>>> [Mon May 6 16:59:44 CST 2024] <<<<<< start DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 27-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker 2 check diff failed 4-th time, retry later check log contain failed 28-th time, retry later wait process dm-worker2 exit... check diff failed 5-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit [Mon May 6 16:59:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 29-th time, retry later rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later check diff failed 6-th time, retry later check log contain failed 30-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 restart dm-master check diff failed 7-th time, retry later check log contain failed 1-th time, retry later wait process dm-master exit... wait process dm-master exit... process dm-master already exit check diff failed 8-th time, retry later dmctl test cmd: "query-status t-ร‹!s`t" check diff successfully [Mon May 6 16:59:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 check diff successfully dmctl test cmd: "stop-task t-ร‹!s`t" [Mon May 6 16:59:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" check diff failed 9-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 restart dm-worker 2 got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker2 exit... check diff failed 10-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit [Mon May 6 17:00:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=3 expected=3 check diff failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" check diff failed 2-th time, retry later [Mon May 6 17:00:02 CST 2024] <<<<<< finish DM-121 pessimistic >>>>>> [Mon May 6 17:00:02 CST 2024] <<<<<< start DM-121 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff failed 3-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:00:04 CST 2024] <<<<<< finish DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> [Mon May 6 17:00:04 CST 2024] <<<<<< start DM-RECOVER_LOCK optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later check diff failed 4-th time, retry later wait process dm-worker.test exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:00:07 CST 2024] <<<<<< finish test_query_timeout >>>>>> [Mon May 6 17:00:07 CST 2024] <<<<<< start test_regexpr_router regexpr-task.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:00:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 1-th time, retry later check diff failed 3-th time, retry later check diff failed 5-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:00:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:00:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time restart dm-master check diff failed 4-th time, retry later check diff failed 6-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" wait process dm-master exit... dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive use sync_diff_inspector to check increment data check diff successfully check diff successfully data checked after one worker was killed try to kill worker port 8263 wait process dm-worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit worker2 was killed dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 [Mon May 6 17:00:11 CST 2024] <<<<<< finish test_multi_task_reduce_and_restart_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" check diff failed 5-th time, retry later wait process dm-master exit... process dm-master already exit wait process dm-master.test exit... wait process dm-master.test exit... check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff failed 6-th time, retry later [Mon May 6 17:00:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check diff failed 7-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:00:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:00:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 8-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:00:19 CST 2024] <<<<<< finish test_regexpr_router regexpr-task.yaml >>>>>> [Mon May 6 17:00:19 CST 2024] <<<<<< start test_regexpr_router regexpr-task-lightning.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" [Mon May 6 17:00:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" restart dm-master wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:00:20 CST 2024] <<<<<< test case ha_cases2 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } wait process dm-master exit... [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } check diff failed 9-th time, retry later [Pipeline] // stage rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:00:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } check log contain failed 1-th time, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:00:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 10-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" check log contain failed 2-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" [Mon May 6 17:00:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" check diff failed at last dmctl test cmd: "stop-task test" [Mon May 6 17:00:25 CST 2024] <<<<<< finish DM-121 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 3-th time, retry later check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 17:00:26 CST 2024] <<<<<< start DM-122 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive check diff successfully dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 17:00:26 CST 2024] <<<<<< finish DM-RECOVER_LOCK optimistic >>>>>> run DM_DropAddColumn case #0 [Mon May 6 17:00:26 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... check log contain failed 4-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later check log contain failed 5-th time, retry later wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:00:30 CST 2024] <<<<<< finish DM-122 pessimistic >>>>>> [Mon May 6 17:00:30 CST 2024] <<<<<< start DM-122 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "shard-ddl-lock" check log contain failed 6-th time, retry later got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:00:32 CST 2024] <<<<<< finish test_regexpr_router regexpr-task-lightning.yaml >>>>>> check log contain failed 7-th time, retry later [Mon May 6 17:00:33 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 2-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:00:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:00:34 CST 2024] <<<<<< finish DM-122 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Mon May 6 17:00:35 CST 2024] <<<<<< start DM-123 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 8-th time, retry later check diff failed 3-th time, retry later [Mon May 6 17:00:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 9-th time, retry later check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:00:38 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #1 [Mon May 6 17:00:38 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" check log contain failed 10-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:00:39 CST 2024] <<<<<< finish DM-123 pessimistic >>>>>> [Mon May 6 17:00:39 CST 2024] <<<<<< start DM-123 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status t-ร‹!s`t" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task /tmp/dm_test/all_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" got=2 expected=2 check diff successfully restart dm-master dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master exit... check log contain failed 11-th time, retry later check diff successfully dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-master exit... process dm-master already exit dmctl test cmd: "pause-task t-ร‹!s`t" dmctl test cmd: "operate-schema set -s mysql-replica-01 t-ร‹!s`t -d all_mode -t no_diff /tmp/dm_test/all_mode/schema.sql" dmctl test cmd: "resume-task t-ร‹!s`t" wait pattern dm-worker1.toml exit... check log contain failed 12-th time, retry later wait pattern dm-worker1.toml exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:00:43 CST 2024] <<<<<< finish DM-123 optimistic >>>>>> wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... [Mon May 6 17:00:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Mon May 6 17:00:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:00:44 CST 2024] <<<<<< start DM-124 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 13-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 wait pattern dm-worker2.toml exit... got=2 expected=2 wait pattern dm-worker2.toml exit... check diff successfully dmctl test cmd: "stop-task test" wait pattern dm-worker2.toml exit... [Mon May 6 17:00:46 CST 2024] <<<<<< finish DM-124 optimistic >>>>>> wait pattern dm-worker2.toml exit... rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... check log contain failed 14-th time, retry later wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Mon May 6 17:00:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 17:00:47 CST 2024] <<<<<< start DM-125 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check diff failed 1-th time, retry later check log contain failed 15-th time, retry later check diff failed 2-th time, retry later check log contain failed 16-th time, retry later check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Mon May 6 17:00:51 CST 2024] <<<<<< finish DM-125 optimistic >>>>>> [Mon May 6 17:00:52 CST 2024] <<<<<< start DM-126 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 17-th time, retry later check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 18-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "shard-ddl-lock" [Mon May 6 17:00:56 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #2 [Mon May 6 17:00:56 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 19-th time, retry later dmctl test cmd: "stop-task test" [Mon May 6 17:00:57 CST 2024] <<<<<< finish DM-126 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 after restart dm-worker, task should resume automatically dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml" check diff successfully restart dm-master [Mon May 6 17:00:58 CST 2024] <<<<<< start DM-127 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 20-th time, retry later wait process dm-master exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later HTTP 127.0.0.1:8261/apis/v1alpha1/status/t-ร‹!s`t is alive wait process dm-master exit... process dm-master already exit check log contain failed 21-th time, retry later check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Mon May 6 17:01:02 CST 2024] <<<<<< finish DM-127 optimistic >>>>>> [Mon May 6 17:01:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 22-th time, retry later wait process tidb-server exit... process tidb-server already exit [Mon May 6 17:01:03 CST 2024] <<<<<< start DM-128 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 23-th time, retry later check diff failed 1-th time, retry later Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check log contain failed 24-th time, retry later check diff successfully dmctl test cmd: "stop-task test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' [Mon May 6 17:01:07 CST 2024] <<<<<< finish DM-128 pessimistic >>>>>> [Mon May 6 17:01:07 CST 2024] <<<<<< start DM-128 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check diff failed 2-th time, retry later check log contain failed 25-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later [Mon May 6 17:01:09 CST 2024] <<<<<< finish DM-128 optimistic >>>>>> [Mon May 6 17:01:10 CST 2024] <<<<<< start DM-129 pessimistic >>>>>> check diff failed 3-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 26-th time, retry later check diff successfully dmctl test cmd: "pause-relay -s mysql-replica-01" dmctl test cmd: "resume-relay -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:01:13 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #3 [Mon May 6 17:01:13 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 27-th time, retry later check diff successfully dmctl test cmd: "stop-task test" relay logs dm-it-548043a1-cbb8-476b-aa5f-604454ab70a7-xr7zl-x7gr9-bin.000002 relay.meta check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/all_mode/worker2/dumped_data.t-ร‹!s`t: No such file or directory worker2 auto removed dump files check no password in log dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully [Mon May 6 17:01:14 CST 2024] <<<<<< finish DM-129 pessimistic >>>>>> [Mon May 6 17:01:14 CST 2024] <<<<<< start DM-129 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 1-th time, retry later check log contain failed 28-th time, retry later matched matched [Mon May 6 17:01:15 CST 2024] <<<<<< start test_source_and_target_with_empty_gtid >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:01:16 CST 2024] <<<<<< finish DM-129 optimistic >>>>>> dmctl test cmd: "shard-ddl-lock" wait process dm-master.test exit... got=1 expected=1 restart dm-master check log contain failed 29-th time, retry later [Mon May 6 17:01:17 CST 2024] <<<<<< start DM-130 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master exit... wait process dm-worker.test exit... wait process dm-master exit... dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 30-th time, retry later process dm-master already exit dmctl test cmd: "stop-task test" [Mon May 6 17:01:19 CST 2024] <<<<<< finish DM-130 pessimistic >>>>>> [Mon May 6 17:01:19 CST 2024] <<<<<< start DM-130 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:01:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 1-th time, retry later [Mon May 6 17:01:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:01:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/all_mode/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" check diff successfully [Mon May 6 17:01:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 gtid is empty start task and check stage dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task-no-gtid.yaml --remove-meta=true" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb1 --action skip" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb2 --action skip" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" [Mon May 6 17:01:24 CST 2024] <<<<<< finish DM-130 optimistic >>>>>> got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff successfully ERROR 1146 (42S02) at line 1: Table 'all_mode.t2' doesn't exist run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed 2-th time, retry later wait process dm-master.test exit... check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later check diff failed 3-th time, retry later got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:01:30 CST 2024] <<<<<< test case shardddl4 success! >>>>>> start running case: [shardddl4_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:01:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 17:01:30 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #4 [Mon May 6 17:01:30 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:01:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source1.yaml" [Mon May 6 17:01:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:01:33 CST 2024] <<<<<< test case duplicate_event success! >>>>>> start running case: [expression_filter] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:01:33 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:01:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master [Mon May 6 17:01:35 CST 2024] <<<<<< start DM-TABLE_CHECKPOINT_BACKWARD optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml " wait process dm-master exit... process dm-master already exit dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again [Mon May 6 17:01:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" ls: cannot access /tmp/dm_test/expression_filter/worker1/schema-tracker*: No such file or directory schema tracker path has been cleaned 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:01:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 3-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:01:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" [Mon May 6 17:01:49 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #5 [Mon May 6 17:01:49 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 restart dm-master wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Mon May 6 17:01:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check diff failed 3-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... check diff failed 2-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:02:03 CST 2024] <<<<<< finish DM-TABLE_CHECKPOINT_BACKWARD optimistic >>>>>> [Mon May 6 17:02:03 CST 2024] <<<<<< start DM-RESYNC_NOT_FLUSHED optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:02:03 CST 2024] <<<<<< test case expression_filter success! >>>>>> start running case: [extend_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: sql [Mon May 6 17:02:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker1 rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:02:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later wait process worker1 exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:02:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" wait process worker1 exit... process worker1 already exit [Mon May 6 17:02:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:02:07 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> [Mon May 6 17:02:07 CST 2024] <<<<<< start DM-ADD_DROP_PARTITIONS optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive restart dm-worker2 dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" wait process worker2 exit... dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process worker2 exit... process worker2 already exit [Mon May 6 17:02:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "stop-task test" [Mon May 6 17:02:13 CST 2024] <<<<<< finish DM-ADD_DROP_PARTITIONS optimistic >>>>>> [Mon May 6 17:02:13 CST 2024] <<<<<< start DM-036 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "stop-task test" [Mon May 6 17:02:17 CST 2024] <<<<<< finish DM-036 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: loader [Mon May 6 17:02:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:02:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:02:18 CST 2024] <<<<<< start DM-037 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:02:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 17:02:19 CST 2024] <<<<<< finish DM-037 pessimistic >>>>>> [Mon May 6 17:02:19 CST 2024] <<<<<< start DM-037 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" [Mon May 6 17:02:21 CST 2024] <<<<<< finish DM-037 optimistic >>>>>> [Mon May 6 17:02:22 CST 2024] <<<<<< start DM-038 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" [Mon May 6 17:02:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:02:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later check diff failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:02:32 CST 2024] <<<<<< test case extend_column success! >>>>>> start running case: [fake_rotate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:02:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 2-th time, retry later rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:02:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/fake_rotate_event/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-task.yaml --remove-meta" check diff failed at last dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check diff successfully kill dm-worker wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 17:02:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:02:39 CST 2024] <<<<<< finish DM-038 pessimistic >>>>>> [Mon May 6 17:02:39 CST 2024] <<<<<< start DM-038 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:02:44 CST 2024] <<<<<< test case fake_rotate_event success! >>>>>> start running case: [foreign_key] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:02:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:02:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/foreign_key/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff failed 1-th time, retry later check diff successfully restart dm-worker1 wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Mon May 6 17:02:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive restart dm-worker2 wait process worker2 exit... wait process dm-master.test exit... wait process worker2 exit... process worker2 already exit [Mon May 6 17:02:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit check diff failed 2-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-task test" [Mon May 6 17:02:52 CST 2024] <<<<<< finish DM-RESYNC_NOT_FLUSHED optimistic >>>>>> [Mon May 6 17:02:52 CST 2024] <<<<<< start DM-RESYNC_TXN_INTERRUPT optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker1 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:02:54 CST 2024] <<<<<< test case foreign_key success! >>>>>> start running case: [full_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:02:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process worker1 exit... check diff failed at last dmctl test cmd: "stop-task test" [Mon May 6 17:02:55 CST 2024] <<<<<< finish DM-038 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:02:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker1 exit... process worker1 already exit [Mon May 6 17:02:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" rpc addr 127.0.0.1:8262 is alive restart dm-worker2 [Mon May 6 17:02:56 CST 2024] <<<<<< start DM-039 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 17:02:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker2 exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process worker2 exit... process worker2 already exit [Mon May 6 17:02:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" rpc addr 127.0.0.1:8263 is alive begin; insert into shardddl1.tb2 values (1,1); insert into shardddl1.tb2 values (2,2); insert into shardddl1.tb2 values (3,3); insert into shardddl1.tb2 values (4,4); insert into shardddl1.tb2 values (5,5); insert into shardddl1.tb2 values (6,6); insert into shardddl1.tb2 values (7,7); insert into shardddl1.tb2 values (8,8); insert into shardddl1.tb2 values (9,9); insert into shardddl1.tb2 values (10,10); commit; begin; insert into shardddl1.t_1 values (11,11); insert into shardddl1.t_1 values (12,12); insert into shardddl1.t_1 values (13,13); insert into shardddl1.t_1 values (14,14); insert into shardddl1.t_1 values (15,15); insert into shardddl1.t_1 values (16,16); insert into shardddl1.t_1 values (17,17); insert into shardddl1.t_1 values (18,18); insert into shardddl1.t_1 values (19,19); insert into shardddl1.t_1 values (20,20); insert into shardddl1.t_1 values (21,21); insert into shardddl1.t_1 values (22,22); insert into shardddl1.t_1 values (23,23); insert into shardddl1.t_1 values (24,24); insert into shardddl1.t_1 values (25,25); insert into shardddl1.t_1 values (26,26); insert into shardddl1.t_1 values (27,27); insert into shardddl1.t_1 values (28,28); insert into shardddl1.t_1 values (29,29); insert into shardddl1.t_1 values (30,30); insert into shardddl1.t_1 values (31,31); insert into shardddl1.t_1 values (32,32); insert into shardddl1.t_1 values (33,33); insert into shardddl1.t_1 values (34,34); insert into shardddl1.t_1 values (35,35); insert into shardddl1.t_1 values (36,36); insert into shardddl1.t_1 values (37,37); insert into shardddl1.t_1 values (38,38); insert into shardddl1.t_1 values (39,39); insert into shardddl1.t_1 values (40,40); insert into shardddl1.t_1 values (41,41); insert into shardddl1.t_1 values (42,42); insert into shardddl1.t_1 values (43,43); insert into shardddl1.t_1 values (44,44); insert into shardddl1.t_1 values (45,45); insert into shardddl1.t_1 values (46,46); insert into shardddl1.t_1 values (47,47); insert into shardddl1.t_1 values (48,48); insert into shardddl1.t_1 values (49,49); insert into shardddl1.t_1 values (50,50); commit; begin; insert into shardddl1.tb1 values (51,51); insert into shardddl1.tb1 values (52,52); insert into shardddl1.tb1 values (53,53); insert into shardddl1.tb1 values (54,54); insert into shardddl1.tb1 values (55,55); insert into shardddl1.tb1 values (56,56); insert into shardddl1.tb1 values (57,57); insert into shardddl1.tb1 values (58,58); insert into shardddl1.tb1 values (59,59); insert into shardddl1.tb1 values (60,60); commit; begin; insert into shardddl1.t_1 values (61,61); insert into shardddl1.t_1 values (62,62); insert into shardddl1.t_1 values (63,63); insert into shardddl1.t_1 values (64,64); insert into shardddl1.t_1 values (65,65); insert into shardddl1.t_1 values (66,66); insert into shardddl1.t_1 values (67,67); insert into shardddl1.t_1 values (68,68); insert into shardddl1.t_1 values (69,69); insert into shardddl1.t_1 values (70,70); commit; check diff failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/full_mode/dm-task.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:00 CST 2024] <<<<<< finish DM-039 pessimistic >>>>>> [Mon May 6 17:03:00 CST 2024] <<<<<< start DM-039 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:02 CST 2024] <<<<<< finish DM-039 optimistic >>>>>> check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 <<<<<< test_source_and_target_with_empty_gtid success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 17:03:03 CST 2024] <<<<<< start DM-040 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" check diff failed 3-th time, retry later wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 17:03:05 CST 2024] <<<<<< finish DM-040 pessimistic >>>>>> [Mon May 6 17:03:05 CST 2024] <<<<<< start DM-040 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:03:06 CST 2024] <<<<<< test case all_mode success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } wait process dm-worker.test exit... [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:03:07 CST 2024] <<<<<< finish DM-040 optimistic >>>>>> [Mon May 6 17:03:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 17:03:08 CST 2024] <<<<<< start DM-041 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:03:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:03:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully restart dm-worker1 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" wait process worker1 exit... process worker1 already exit [Mon May 6 17:03:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/full_mode/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive restart dm-worker2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:14 CST 2024] <<<<<< finish DM-041 pessimistic >>>>>> [Mon May 6 17:03:14 CST 2024] <<<<<< start DM-041 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait process worker2 exit... [Mon May 6 17:03:15 CST 2024] <<<<<< finish DM-041 optimistic >>>>>> wait process worker2 exit... process worker2 already exit check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/full_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/full_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 17:03:16 CST 2024] <<<<<< start DM-043 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 17:03:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 17:03:18 CST 2024] <<<<<< finish DM-043 pessimistic >>>>>> [Mon May 6 17:03:18 CST 2024] <<<<<< start DM-043 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 17:03:20 CST 2024] <<<<<< finish DM-043 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 17:03:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8263 alive the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:03:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Mon May 6 17:03:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... begin; insert into shardddl1.tb2 values (101,101); insert into shardddl1.tb2 values (102,102); insert into shardddl1.tb2 values (103,103); insert into shardddl1.tb2 values (104,104); insert into shardddl1.tb2 values (105,105); insert into shardddl1.tb2 values (106,106); insert into shardddl1.tb2 values (107,107); insert into shardddl1.tb2 values (108,108); insert into shardddl1.tb2 values (109,109); insert into shardddl1.tb2 values (110,110); commit; begin; insert into shardddl1.tb1 values (111,111); insert into shardddl1.tb1 values (112,112); insert into shardddl1.tb1 values (113,113); insert into shardddl1.tb1 values (114,114); insert into shardddl1.tb1 values (115,115); insert into shardddl1.tb1 values (116,116); insert into shardddl1.tb1 values (117,117); insert into shardddl1.tb1 values (118,118); insert into shardddl1.tb1 values (119,119); insert into shardddl1.tb1 values (120,120); commit; begin; insert into shardddl1.tb2 values (121,121); insert into shardddl1.tb2 values (122,122); insert into shardddl1.tb2 values (123,123); insert into shardddl1.tb2 values (124,124); insert into shardddl1.tb2 values (125,125); insert into shardddl1.tb2 values (126,126); insert into shardddl1.tb2 values (127,127); insert into shardddl1.tb2 values (128,128); insert into shardddl1.tb2 values (129,129); insert into shardddl1.tb2 values (130,130); commit; begin; insert into shardddl1.t_1 values (131,131); insert into shardddl1.t_1 values (132,132); insert into shardddl1.t_1 values (133,133); insert into shardddl1.t_1 values (134,134); insert into shardddl1.t_1 values (135,135); insert into shardddl1.t_1 values (136,136); insert into shardddl1.t_1 values (137,137); insert into shardddl1.t_1 values (138,138); insert into shardddl1.t_1 values (139,139); insert into shardddl1.t_1 values (140,140); commit; check diff successfully begin; insert into shardddl1.tb2 values (201,201); insert into shardddl1.tb2 values (202,202); insert into shardddl1.tb2 values (203,203); insert into shardddl1.tb2 values (204,204); insert into shardddl1.tb2 values (205,205); insert into shardddl1.tb2 values (206,206); insert into shardddl1.tb2 values (207,207); insert into shardddl1.tb2 values (208,208); insert into shardddl1.tb2 values (209,209); insert into shardddl1.tb2 values (210,210); commit; begin; insert into shardddl1.tb1 values (211,211); insert into shardddl1.tb1 values (212,212); insert into shardddl1.tb1 values (213,213); insert into shardddl1.tb1 values (214,214); insert into shardddl1.tb1 values (215,215); insert into shardddl1.tb1 values (216,216); insert into shardddl1.tb1 values (217,217); insert into shardddl1.tb1 values (218,218); insert into shardddl1.tb1 values (219,219); insert into shardddl1.tb1 values (220,220); commit; begin; insert into shardddl1.tb2 values (221,221); insert into shardddl1.tb2 values (222,222); insert into shardddl1.tb2 values (223,223); insert into shardddl1.tb2 values (224,224); insert into shardddl1.tb2 values (225,225); insert into shardddl1.tb2 values (226,226); insert into shardddl1.tb2 values (227,227); insert into shardddl1.tb2 values (228,228); insert into shardddl1.tb2 values (229,229); insert into shardddl1.tb2 values (230,230); commit; begin; insert into shardddl1.t_1 values (231,231); insert into shardddl1.t_1 values (232,232); insert into shardddl1.t_1 values (233,233); insert into shardddl1.t_1 values (234,234); insert into shardddl1.t_1 values (235,235); insert into shardddl1.t_1 values (236,236); insert into shardddl1.t_1 values (237,237); insert into shardddl1.t_1 values (238,238); insert into shardddl1.t_1 values (239,239); insert into shardddl1.t_1 values (240,240); commit; check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:03:26 CST 2024] <<<<<< test case shardddl2 success! >>>>>> start running case: [shardddl2_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:03:26 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" check diff successfully wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 17:03:27 CST 2024] <<<<<< finish DM-RESYNC_TXN_INTERRUPT optimistic >>>>>> [Mon May 6 17:03:27 CST 2024] <<<<<< start DM-STRICT_OPTIMISTIC_SINGLE_SOURCE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/single-source-strict-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:03:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 17:03:30 CST 2024] <<<<<< finish DM-STRICT_OPTIMISTIC_SINGLE_SOURCE optimistic >>>>>> [Mon May 6 17:03:30 CST 2024] <<<<<< start DM-STRICT_OPTIMISTIC_DOUBLE_SOURCE optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-strict-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2_1/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Mon May 6 17:03:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 17:03:32 CST 2024] <<<<<< finish DM-STRICT_OPTIMISTIC_DOUBLE_SOURCE optimistic >>>>>> [Mon May 6 17:03:32 CST 2024] <<<<<< start DM-131 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2_1/source2.yaml" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully [Mon May 6 17:03:33 CST 2024] <<<<<< start DM-046 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:03:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 17:03:34 CST 2024] <<<<<< finish DM-131 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:03:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:03:35 CST 2024] <<<<<< start DM-132 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 17:03:35 CST 2024] <<<<<< finish DM-046 pessimistic >>>>>> [Mon May 6 17:03:35 CST 2024] <<<<<< start DM-046 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" [Mon May 6 17:03:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" [Mon May 6 17:03:37 CST 2024] <<<<<< finish DM-046 optimistic >>>>>> [Mon May 6 17:03:38 CST 2024] <<<<<< start DM-047 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task-2.yaml --remove-meta" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed 1-th time, retry later wait process dm-master.test exit... got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:42 CST 2024] <<<<<< finish DM-047 pessimistic >>>>>> [Mon May 6 17:03:42 CST 2024] <<<<<< start DM-047 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:44 CST 2024] <<<<<< finish DM-132 pessimistic >>>>>> [Mon May 6 17:03:44 CST 2024] <<<<<< start DM-132 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 17:03:44 CST 2024] <<<<<< finish DM-047 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:03:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 17:03:45 CST 2024] <<<<<< start DM-048 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:46 CST 2024] <<<<<< finish DM-132 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:03:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Mon May 6 17:03:47 CST 2024] <<<<<< start DM-133 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:03:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/full_mode/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:49 CST 2024] <<<<<< finish DM-048 pessimistic >>>>>> [Mon May 6 17:03:49 CST 2024] <<<<<< start DM-048 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/full_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" [Mon May 6 17:03:51 CST 2024] <<<<<< finish DM-048 optimistic >>>>>> got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/full_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/full_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 17:03:51 CST 2024] <<<<<< finish DM-133 pessimistic >>>>>> [Mon May 6 17:03:51 CST 2024] <<<<<< start DM-133 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 17:03:52 CST 2024] <<<<<< start DM-049 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 17:03:54 CST 2024] <<<<<< finish DM-133 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... [Mon May 6 17:03:55 CST 2024] <<<<<< start DM-134 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:03:56 CST 2024] <<<<<< test case full_mode success! >>>>>> start running case: [gbk] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/run.sh... Verbose mode = false check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:57 CST 2024] <<<<<< finish DM-049 pessimistic >>>>>> [Mon May 6 17:03:57 CST 2024] <<<<<< start DM-049 optimistic >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 17:03:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:03:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:03:59 CST 2024] <<<<<< finish DM-134 pessimistic >>>>>> [Mon May 6 17:03:59 CST 2024] <<<<<< start DM-134 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:03:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gbk/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/gbk/source2.yaml" [Mon May 6 17:04:01 CST 2024] <<<<<< finish DM-134 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" prepare data start task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-task.yaml --remove-meta" [Mon May 6 17:04:02 CST 2024] <<<<<< finish DM-049 optimistic >>>>>> [Mon May 6 17:04:02 CST 2024] <<<<<< start DM-135 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 17:04:03 CST 2024] <<<<<< start DM-050 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully prepare incremental data check incremental phase check diff successfully ERROR 1146 (42S02) at line 1: Table 'gbk.ddl1' doesn't exist run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Mon May 6 17:04:04 CST 2024] <<<<<< finish DM-135 pessimistic >>>>>> [Mon May 6 17:04:04 CST 2024] <<<<<< start DM-135 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Mon May 6 17:04:05 CST 2024] <<<<<< finish DM-050 pessimistic >>>>>> [Mon May 6 17:04:05 CST 2024] <<<<<< start DM-050 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" ERROR 1146 (42S02) at line 1: Table 'gbk.ddl2_copy' doesn't exist run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 17:04:07 CST 2024] <<<<<< finish DM-050 optimistic >>>>>> [Mon May 6 17:04:07 CST 2024] <<<<<< finish DM-135 optimistic >>>>>> ERROR 1146 (42S02) at line 1: Table 'gbk.ddl3' doesn't exist run tidb sql failed 1-th time, retry later [Mon May 6 17:04:08 CST 2024] <<<<<< start DM-051 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 17:04:08 CST 2024] <<<<<< start DM-136 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" ERROR 1049 (42000) at line 1: Unknown database 'gbk3' run tidb sql failed 1-th time, retry later [Mon May 6 17:04:10 CST 2024] <<<<<< finish DM-051 pessimistic >>>>>> [Mon May 6 17:04:10 CST 2024] <<<<<< start DM-051 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 17:04:12 CST 2024] <<<<<< finish DM-051 optimistic >>>>>> ERROR 1146 (42S02) at line 1: Table 'gbk.ddl4' doesn't exist run tidb sql failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:12 CST 2024] <<<<<< finish DM-136 optimistic >>>>>> [Mon May 6 17:04:13 CST 2024] <<<<<< start DM-056 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 17:04:13 CST 2024] <<<<<< start DM-137 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" prepare data for invalid connection test check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Mon May 6 17:04:15 CST 2024] <<<<<< finish DM-056 pessimistic >>>>>> [Mon May 6 17:04:15 CST 2024] <<<<<< start DM-056 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:18 CST 2024] <<<<<< finish DM-137 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:04:19 CST 2024] <<<<<< start DM-138 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:20 CST 2024] <<<<<< finish DM-056 optimistic >>>>>> dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive start test invalid connection with status running check count check diff successfully check test invalid connection with status running successfully got=2 expected=2 check diff failed 1-th time, retry later [Mon May 6 17:04:21 CST 2024] <<<<<< start DM-057 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:23 CST 2024] <<<<<< finish DM-138 pessimistic >>>>>> [Mon May 6 17:04:23 CST 2024] <<<<<< start DM-138 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:25 CST 2024] <<<<<< finish DM-138 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:25 CST 2024] <<<<<< finish DM-057 pessimistic >>>>>> rpc addr 127.0.0.1:8263 is alive start test invalid connection with status queueing check count check diff successfully check test invalid connection with status queueing successfully [Mon May 6 17:04:26 CST 2024] <<<<<< start DM-058 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" [Mon May 6 17:04:26 CST 2024] <<<<<< start DM-139 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later [Mon May 6 17:04:28 CST 2024] <<<<<< finish DM-058 pessimistic >>>>>> [Mon May 6 17:04:28 CST 2024] <<<<<< start DM-058 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive start test invalid connection with status none check count got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" check diff successfully check test invalid connection with status none successfully [Mon May 6 17:04:30 CST 2024] <<<<<< finish DM-058 optimistic >>>>>> [Mon May 6 17:04:30 CST 2024] <<<<<< finish DM-139 pessimistic >>>>>> [Mon May 6 17:04:30 CST 2024] <<<<<< start DM-139 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... [Mon May 6 17:04:31 CST 2024] <<<<<< start DM-059 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 17:04:32 CST 2024] <<<<<< finish DM-139 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 17:04:33 CST 2024] <<<<<< start DM-142 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive start test inserting data after invalid connection check count check diff successfully check test inserting data after invalid connection successfully wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:37 CST 2024] <<<<<< finish DM-059 pessimistic >>>>>> [Mon May 6 17:04:37 CST 2024] <<<<<< start DM-059 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:38 CST 2024] <<<<<< finish DM-142 pessimistic >>>>>> rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 [Mon May 6 17:04:39 CST 2024] <<<<<< start DM-143 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start test adding UNIQUE on column with duplicate data check cancelled error dmctl test cmd: "query-status gbk" got=0 expected=1 command: query-status gbk origin SQL: \[ALTER TABLE gbk.invalid_conn_test1 ADD UNIQUE(i)\]: DDL ALTER TABLE `gbk`.`invalid_conn_test1` ADD UNIQUE(`i`) executed in background and met error count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later [Mon May 6 17:04:41 CST 2024] <<<<<< finish DM-143 pessimistic >>>>>> [Mon May 6 17:04:42 CST 2024] <<<<<< start DM-145 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" got=1 expected=1 dmctl test cmd: "resume-task gbk" got=3 expected=3 check test adding UNIQUE on column with duplicate data successfully check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Mon May 6 17:04:43 CST 2024] <<<<<< finish DM-059 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:04:44 CST 2024] <<<<<< start DM-062 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later [Mon May 6 17:04:46 CST 2024] <<<<<< finish DM-145 pessimistic >>>>>> [Mon May 6 17:04:46 CST 2024] <<<<<< start DM-145 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start test invalid connection with status running (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:48 CST 2024] <<<<<< finish DM-145 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:48 CST 2024] <<<<<< finish DM-062 pessimistic >>>>>> [Mon May 6 17:04:48 CST 2024] <<<<<< start DM-062 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" check count 2 check diff successfully check test invalid connection with status running (multi-schema change) successfully [Mon May 6 17:04:49 CST 2024] <<<<<< start DM-146 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:04:50 CST 2024] <<<<<< finish DM-062 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:04:51 CST 2024] <<<<<< finish DM-146 pessimistic >>>>>> [Mon May 6 17:04:51 CST 2024] <<<<<< start DM-146 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 17:04:51 CST 2024] <<<<<< start DM-063 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive start test invalid connection with status queueing (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 17:04:52 CST 2024] <<<<<< finish DM-063 pessimistic >>>>>> [Mon May 6 17:04:52 CST 2024] <<<<<< start DM-063 optimistic >>>>>> [Mon May 6 17:04:53 CST 2024] <<<<<< finish DM-146 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 17:04:54 CST 2024] <<<<<< start DM-147 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later check count 2 check diff successfully check test invalid connection with status queueing (multi-schema change) successfully dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl4_1/schema.sql -s mysql-replica-01" dmctl test cmd: "binlog replace test "alter table shardddl1.tb1 drop column b"" wait process dm-worker.test exit... got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:04:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 1-th time, retry later [Mon May 6 17:04:56 CST 2024] <<<<<< finish DM-147 optimistic >>>>>> [Mon May 6 17:04:57 CST 2024] <<<<<< start DM-148 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:04:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "stop-task test" [Mon May 6 17:04:58 CST 2024] <<<<<< finish DM-063 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive start test invalid connection with status none (multi-schema change) check count 1 check count 2 check diff successfully check test invalid connection with status none (multi-schema change) successfully check diff failed 1-th time, retry later [Mon May 6 17:04:59 CST 2024] <<<<<< start DM-064 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:01 CST 2024] <<<<<< finish DM-148 pessimistic >>>>>> [Mon May 6 17:05:01 CST 2024] <<<<<< start DM-148 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:05:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:05:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 17:05:03 CST 2024] <<<<<< finish DM-148 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:03 CST 2024] <<<<<< finish DM-064 pessimistic >>>>>> [Mon May 6 17:05:03 CST 2024] <<<<<< start DM-064 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start test inserting data after invalid connection (multi-schema change) check count 1 run tidb sql failed 1-th time, retry later [Mon May 6 17:05:04 CST 2024] <<<<<< start DM-149 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" [Mon May 6 17:05:05 CST 2024] <<<<<< finish DM-064 optimistic >>>>>> check count 2 check diff successfully check test inserting data after invalid connection (multi-schema change) successfully got=2 expected=2 [Mon May 6 17:05:06 CST 2024] <<<<<< start DM-065 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" check diff failed 1-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:05:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:05:08 CST 2024] <<<<<< finish DM-065 pessimistic >>>>>> [Mon May 6 17:05:08 CST 2024] <<<<<< start DM-065 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:05:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gbk/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:09 CST 2024] <<<<<< finish DM-149 pessimistic >>>>>> [Mon May 6 17:05:09 CST 2024] <<<<<< start DM-149 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive start test adding UNIQUE on column with duplicate data (multi-schema change) check cancelled error dmctl test cmd: "query-status gbk" got=0 expected=1 command: query-status gbk origin SQL: \[ALTER TABLE gbk.invalid_conn_test1 ADD UNIQUE(k), ADD UNIQUE(m)\]: DDL ALTER TABLE `gbk`.`invalid_conn_test1` ADD UNIQUE(`k`) executed in background and met error count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:11 CST 2024] <<<<<< finish DM-149 optimistic >>>>>> [Mon May 6 17:05:12 CST 2024] <<<<<< start DM-150 pessimistic >>>>>> dmctl test cmd: "stop-task test" [Mon May 6 17:05:12 CST 2024] <<<<<< finish DM-065 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" got=1 expected=1 check test adding UNIQUE on column with duplicate data (multi-schema change) successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 17:05:13 CST 2024] <<<<<< start DM-066 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "show-ddl-locks" got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:16 CST 2024] <<<<<< finish DM-150 pessimistic >>>>>> [Mon May 6 17:05:16 CST 2024] <<<<<< start DM-150 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:17 CST 2024] <<<<<< finish DM-066 pessimistic >>>>>> [Mon May 6 17:05:17 CST 2024] <<<<<< start DM-066 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:05:17 CST 2024] <<<<<< test case gbk success! >>>>>> start running case: [gtid] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:05:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Mon May 6 17:05:18 CST 2024] <<<<<< finish DM-150 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 17:05:19 CST 2024] <<<<<< finish DM-066 optimistic >>>>>> [Mon May 6 17:05:19 CST 2024] <<<<<< start DM-151 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:05:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Mon May 6 17:05:20 CST 2024] <<<<<< start DM-067 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source1.yaml" got=2 expected=2 dmctl test cmd: "show-ddl-locks" got=1 expected=1 [Mon May 6 17:05:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Mon May 6 17:05:22 CST 2024] <<<<<< finish DM-067 pessimistic >>>>>> [Mon May 6 17:05:22 CST 2024] <<<<<< start DM-067 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Mon May 6 17:05:24 CST 2024] <<<<<< finish DM-067 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:24 CST 2024] <<<<<< finish DM-151 pessimistic >>>>>> [Mon May 6 17:05:24 CST 2024] <<<<<< start DM-151 optimistic >>>>>> check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "resume-task test" [Mon May 6 17:05:25 CST 2024] <<<<<< start DM-068 pessimistic >>>>>> check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:05:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:05:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:32 CST 2024] <<<<<< finish DM-151 optimistic >>>>>> [Mon May 6 17:05:33 CST 2024] <<<<<< start DM-152 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" [Mon May 6 17:05:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/gtid/source2.yaml" [Mon May 6 17:05:35 CST 2024] <<<<<< finish DM-068 pessimistic >>>>>> [Mon May 6 17:05:35 CST 2024] <<<<<< start DM-068 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml --remove-meta" check diff successfully check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully new_gtid1 b3cad847-0b85-11ef-8754-d6df5ec14d97:6 new_gtid2 b41b494a-0b85-11ef-9114-d6df5ec14d97:6 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> [Mon May 6 17:05:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:38 CST 2024] <<<<<< finish DM-068 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml" check diff successfully check diff failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" check diff successfully check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... [Mon May 6 17:05:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker1.toml >>>>>> [Mon May 6 17:05:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/gtid/conf/dm-task.yaml" check diff successfully check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:05:44 CST 2024] <<<<<< test case shardddl2_1 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } dmctl test cmd: "query-status test" [Pipeline] // dir got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } check diff successfully dmctl test cmd: "stop-task test" [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process dm-master.test exit... [Pipeline] // stage [Pipeline] } wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... [Mon May 6 17:05:48 CST 2024] <<<<<< finish DM-152 optimistic >>>>>> wait process dm-worker.test exit... [Mon May 6 17:05:49 CST 2024] <<<<<< start DM-153 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:05:50 CST 2024] <<<<<< test case gtid success! >>>>>> start running case: [ha_cases] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:05:50 CST 2024] <<<<<< start test_exclusive_relay >>>>>> start DM worker and master cluster [Mon May 6 17:05:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-standalone.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:05:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 -s mysql-replica-01 --from-target" dmctl test cmd: "binlog replace test "alter table shardddl1.tb1 drop column b"" got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:05:52 CST 2024] <<<<<< finish DM-153 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" [Mon May 6 17:05:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Mon May 6 17:05:53 CST 2024] <<<<<< start DM-154 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1 worker2" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 -s mysql-replica-01 --from-source" dmctl test cmd: "binlog skip test" got=3 expected=3 dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" dmctl test cmd: "list-member --worker" got=2 expected=2 got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source show -s mysql-replica-02" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Mon May 6 17:05:55 CST 2024] <<<<<< finish DM-154 optimistic >>>>>> wait process dm-master.test exit... [Mon May 6 17:05:56 CST 2024] <<<<<< start DM-155 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... restart worker1 restart dm-worker1 wait process dm-worker.test exit... wait process worker1 exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table wait process worker1 exit... process worker1 already exit [Mon May 6 17:06:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:06:02 CST 2024] <<<<<< finish test_exclusive_relay >>>>>> [Mon May 6 17:06:02 CST 2024] <<<<<< start test_exclusive_relay_2 >>>>>> start DM worker and master cluster [Mon May 6 17:06:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-standalone.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time restart worker2 restart dm-worker2 rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:06:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" [Mon May 6 17:06:04 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker2 exit... process worker2 already exit [Mon May 6 17:06:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1" restart master restart dm-master wait process dm-master exit... got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=2 expected=2 [Mon May 6 17:06:09 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8264 is alive kill dm-worker1 [Mon May 6 17:06:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --name worker3" got=1 expected=1 [Mon May 6 17:06:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member --name worker3" got=1 expected=1 dmctl test cmd: "list-member --name worker1" got=1 expected=1 kill dm-worker2 rpc addr 127.0.0.1:8261 is alive wait process dm-worker2 exit... restart worker1 restart dm-worker1 wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "operate-source show -s mysql-replica-02" got=1 expected=1 [Mon May 6 17:06:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker1 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker2" got=1 expected=1 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process worker1 exit... wait process dm-master.test exit... wait process worker1 exit... wait process dm-master.test exit... process dm-master.test already exit wait process worker1 exit... wait process worker1 exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process worker1 exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table wait process worker1 exit... wait process worker1 exit... [Mon May 6 17:06:24 CST 2024] <<<<<< finish test_exclusive_relay_2 >>>>>> [Mon May 6 17:06:24 CST 2024] <<<<<< start test_last_bound >>>>>> [Mon May 6 17:06:24 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Mon May 6 17:06:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive import prepare data start DM worker and master cluster [Mon May 6 17:06:26 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master1.toml >>>>>> [Mon May 6 17:06:26 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master2.toml >>>>>> [Mon May 6 17:06:26 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time restart master restart dm-master wait process dm-master exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Mon May 6 17:06:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" [Mon May 6 17:06:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Mon May 6 17:06:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" rpc addr 127.0.0.1:8261 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-task.yaml " restart master restart dm-master dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase wait process dm-master exit... wait process dm-master exit... process dm-master already exit use sync_diff_inspector to check increment data check diff successfully [Mon May 6 17:06:39 CST 2024] <<<<<< finish test_running >>>>>> worker1bound "mysql-replica-01" worker2bound "mysql-replica-02" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" [Mon May 6 17:06:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 kill dm-worker1 rpc addr 127.0.0.1:8261 is alive wait process dm-worker1 exit... restart worker2 restart dm-worker2 wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process worker2 exit... wait process dm-worker2 exit... wait process worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker1 [Mon May 6 17:06:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8262 is alive start worker2 [Mon May 6 17:06:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker1 --name worker2" got=1 expected=1 got=1 expected=1 kill dm-worker1 wait process worker2 exit... wait process dm-worker1 exit... wait process worker2 exit... wait process worker2 exit... wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process dm-worker2 exit... wait process worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker2 [Mon May 6 17:06:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker2 exit... process worker2 already exit [Mon May 6 17:06:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive rpc addr 127.0.0.1:8263 is alive start worker1 [Mon May 6 17:06:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time restart master restart dm-master rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member --name worker2 --name worker1" got=1 expected=1 got=1 expected=1 kill dm-worker1 wait process dm-worker1 exit... wait process dm-master exit... wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process dm-master exit... process dm-master already exit wait process dm-worker2 exit... [Mon May 6 17:06:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member --name worker1 --name worker2" got=2 expected=2 start worker3 [Mon May 6 17:06:59 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8264 is alive start worker4 [Mon May 6 17:07:00 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "list-member --name worker3 --name worker4" got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later dmctl test cmd: "start-relay -s mysql-replica-01 worker3" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker4" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 check diff successfully kill dm-worker3 check log contain failed 1-th time, retry later wait process dm-worker3 exit... check log contain failed 1-th time, retry later wait process dm-worker3 exit... process dm-worker3 already exit kill dm-worker4 wait process dm-worker4 exit... restart worker2 restart dm-worker2 wait process dm-worker4 exit... process dm-worker4 already exit dmctl test cmd: "list-member --name worker3 --name worker4" got=2 expected=2 start worker1 [Mon May 6 17:07:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8262 is alive start worker2 [Mon May 6 17:07:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --name worker1 --name worker2" got=1 expected=1 got=1 expected=1 num1 1 num2 2 [Mon May 6 17:07:10 CST 2024] <<<<<< finish test_last_bound >>>>>> [Mon May 6 17:07:10 CST 2024] <<<<<< start test_config_name >>>>>> [Mon May 6 17:07:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join1.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:07:10 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/ha_cases/dm-master-join2.toml >>>>>> wait process worker2 exit... wait process worker2 exit... check log contain failed 1-th time (file not exist), retry later [Mon May 6 17:07:12 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/ha_cases/dm-master-join2.toml >>>>>> wait process worker2 exit... process worker2 already exit [Mon May 6 17:07:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> rpc addr 127.0.0.1:8361 is alive [Mon May 6 17:07:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:07:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/ha_cases/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later [Mon May 6 17:07:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/ha_cases/dm-worker2.toml >>>>>> rpc addr 127.0.0.1:8263 is alive [Mon May 6 17:07:15 CST 2024] <<<<<< finish test_config_name >>>>>> [Mon May 6 17:07:15 CST 2024] <<<<<< start test_join_masters_and_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive check log contain failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... check log contain failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... restart master restart dm-master wait process dm-master.test exit... wait process dm-master exit... wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 17:07:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit clean source table wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Mon May 6 17:07:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive query-status from unique master dmctl test cmd: "query-status" got=1 expected=1 [Mon May 6 17:07:28 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time check log contain failed 1-th time, retry later check log contain failed 1-th time, retry later restart worker1 restart dm-worker1 wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... wait process worker1 exit... wait for rpc addr 127.0.0.1:8361 alive the 2-th time wait process worker1 exit... process worker1 already exit [Mon May 6 17:07:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8262 is alive restart worker1 restart dm-worker1 wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Mon May 6 17:07:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:07:44 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time rpc addr 127.0.0.1:8461 is alive [Mon May 6 17:07:50 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time rpc addr 127.0.0.1:8561 is alive [Mon May 6 17:07:56 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master-join5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status" got=1 expected=1 dmctl test cmd: "query-status" got=1 expected=1 dmctl test cmd: "query-status" got=1 expected=1 dmctl test cmd: "query-status" got=1 expected=1 join worker with dm-master1 endpoint [Mon May 6 17:07:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker-join2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker --name=worker2" got=1 expected=1 kill dm-master-join1 wait process dm-master-join1 exit... check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:08:01 CST 2024] <<<<<< finish DM-155 optimistic >>>>>> wait process dm-master-join1 exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master-join1 exit... process dm-master-join1 already exit dmctl test cmd: "list-member --worker --name=worker2" got=1 expected=1 join worker with 5 masters endpoint [Mon May 6 17:08:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8262 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8262 alive the 3-th time wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8262 alive the 4-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive query-status from master2 dmctl test cmd: "query-status" got=1 expected=1 [Mon May 6 17:08:07 CST 2024] <<<<<< finish test_join_masters_and_worker >>>>>> [Mon May 6 17:08:07 CST 2024] <<<<<< start test_standalone_running >>>>>> 4 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:08:07 CST 2024] <<<<<< test case shardddl4_1 success! >>>>>> start running case: [sharding] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:08:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:08:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding/source1.yaml" [Mon May 6 17:08:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding/source2.yaml" wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check sync diff for full dump and load check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" check sync diff for the first increment replication check diff failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... check diff successfully check sync diff for the second increment replication check diff successfully check sync diff for the third increment replication check diff successfully checksum before drop/truncate: checksum: 2273109362, checksum after drop/truncate: checksum: 2273109362 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "stop-task test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:08:22 CST 2024] <<<<<< test case sharding success! >>>>>> start running case: [sequence_sharding] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:08:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:08:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:08:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding/source1.yaml" wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding/source2.yaml" wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-task.yaml " wait process dm-master.test exit... check diff successfully wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... check diff successfully dmctl test cmd: "query-status sequence_sharding" got=0 expected=2 command: query-status sequence_sharding detect inconsistent DDL sequence count: 0 != expected: 2, failed the 0-th time, will retry again wait process dm-master.test exit... wait process dm-master.test exit... got=2 expected=2 wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "resume-task sequence_sharding" dmctl test cmd: "query-status sequence_sharding" got=2 expected=2 dmctl test cmd: "stop-task sequence_sharding" dmctl test cmd: "start-task /tmp/dm_test/sequence_sharding/task.yaml" dmctl test cmd: "query-status sequence_sharding" wait process dm-master.test exit... got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:08:52 CST 2024] <<<<<< test case sequence_sharding success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process dm-master.test exit... [Pipeline] // stage [Pipeline] } wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master standalone cluster [Mon May 6 17:09:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master1.toml >>>>>> [Mon May 6 17:09:11 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master2.toml >>>>>> [Mon May 6 17:09:11 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Mon May 6 17:09:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source1.yaml" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task.yaml " use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task2.yaml" [Mon May 6 17:09:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task2.yaml" dmctl test cmd: "query-status" got=2 expected=2 kill worker2 wait process dm-worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test2" got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases/conf/standalone-task2.yaml" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 [Mon May 6 17:09:25 CST 2024] <<<<<< finish test_standalone_running >>>>>> 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:10:27 CST 2024] <<<<<< test case ha_cases success! >>>>>> start running case: [http_proxies] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit test dm grpc proxy env setting checking for http_proxy=http://127.0.0.1:8080 [Mon May 6 17:10:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected [Mon May 6 17:10:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit test dm grpc proxy env setting checking for https_proxy=https://127.0.0.1:8080 [Mon May 6 17:10:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected [Mon May 6 17:10:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit test dm grpc proxy env setting checking for no_proxy=localhost,127.0.0.1 [Mon May 6 17:10:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected [Mon May 6 17:10:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_proxies/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive ./tests/_utils/check_log_contains: line 15: [: proxy: integer expression expected ./tests/_utils/check_log_contains: line 21: [: proxy: integer expression expected dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:10:58 CST 2024] <<<<<< test case http_proxies success! >>>>>> start running case: [lightning_load_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM master, workers and sources [Mon May 6 17:10:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:11:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_load_task/source1.yaml" [Mon May 6 17:11:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_load_task/source2.yaml" [Mon May 6 17:11:03 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8264 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task2.yaml --remove-meta" dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 got=1 expected=1 test worker restart wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 [Mon May 6 17:11:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 got=1 expected=1 test_transfer_two_sources wait process dm-worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member -w -n worker3" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task3.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status load_task3" got=1 expected=1 [Mon May 6 17:11:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-task4.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "query-status load_task4" got=1 expected=1 [Mon May 6 17:11:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 wait process dm-worker3 exit... wait process dm-worker3 exit... process dm-worker3 already exit dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status load_task1" got=1 expected=1 dmctl test cmd: "query-status load_task2" got=1 expected=1 dmctl test cmd: "list-member -w -n worker1" got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 2-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 3-th time, will retry again got=1 expected=1 got=0 expected=1 command: list-member -w -n worker1 "source": "mysql-replica-01" count: 0 != expected: 1, failed the 4-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status" got=3 expected=3 got=1 expected=1 [Mon May 6 17:11:34 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_load_task/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "list-member -w -n worker2" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status" got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 0-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 1-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 2-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 3-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 4-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 5-th time, will retry again got=3 expected=4 command: query-status "taskStatus": "Running" count: 3 != expected: 4, failed the 6-th time, will retry again got=4 expected=4 check diff successfully check diff successfully check diff successfully check diff successfully 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:11:56 CST 2024] <<<<<< test case lightning_load_task success! >>>>>> start running case: [lightning_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit Starting PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:2379; Connection refused 2024-05-06 17:11:56.473079 W | pkg/fileutil: check file permission: directory "/tmp/dm_test/lightning_mode/pd" exist, but the permission is "drwxr-xr-x". The recommended permission is "-rwx------" to prevent possible unprivileged access to the data. % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 99627 0 --:--:-- --:--:-- --:--:-- 104k % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 102k 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 117 100 117 0 0 215k 0 --:--:-- --:--:-- --:--:-- 114k {"connections":0,"version":"8.0.11-TiDB-v7.5.1-48-g066b97f741","git_hash":"066b97f741e6fbef0740e073940cea08d13b3f43"}[Mon May 6 17:12:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:12:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_mode/source1.yaml" [Mon May 6 17:12:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/lightning_mode/source2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-task.yaml" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Mon May 6 17:12:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-task-dup.yaml --remove-meta" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "stage": "Paused" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "stage": "Paused" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "stage": "Paused" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "stage": "Paused" count: 0 != expected: 1, failed the 3-th time, will retry again got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=0 expected=2 command: query-status test unit": "Sync" count: 0 != expected: 2, failed the 0-th time, will retry again got=1 expected=2 command: query-status test unit": "Sync" count: 1 != expected: 2, failed the 1-th time, will retry again got=1 expected=2 command: query-status test unit": "Sync" count: 1 != expected: 2, failed the 2-th time, will retry again got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/lightning_mode/dm-task-dup.yaml --remove-meta" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test checksum mismatched, KV number in source files: 6, KV number in TiDB cluster: 3 count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test checksum mismatched, KV number in source files: 6, KV number in TiDB cluster: 3 count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test checksum mismatched, KV number in source files: 6, KV number in TiDB cluster: 3 count: 0 != expected: 1, failed the 2-th time, will retry again got=1 expected=1 got=2 expected=2 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=0 expected=2 command: query-status test unit": "Sync" count: 0 != expected: 2, failed the 0-th time, will retry again got=0 expected=2 command: query-status test unit": "Sync" count: 0 != expected: 2, failed the 1-th time, will retry again got=1 expected=2 command: query-status test unit": "Sync" count: 1 != expected: 2, failed the 2-th time, will retry again got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/lightning_mode/conf/dm-task.yaml --remove-meta" check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff successfully check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/lightning_mode/worker1/dumped_data.test: No such file or directory worker1 auto removed dump files ls: cannot access /tmp/dm_test/lightning_mode/worker2/dumped_data.test: No such file or directory worker2 auto removed dump files check no password in log Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:13:15 CST 2024] <<<<<< test case lightning_mode success! >>>>>> start running case: [metrics] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check dashboard data source check dashboard data source success [Mon May 6 17:13:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:13:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/metrics/source1.yaml" [Mon May 6 17:13:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/metrics/source2.yaml" dmctl test cmd: "start-task /tmp/dm_test/metrics/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=3 expected=3 wait for valid metric of dm_syncer_binlog_file{node="syncer",source_id="mysql-replica-01",task="test"} for 1-th time, got value: '1' wait for valid metric of dm_syncer_binlog_file{node="master",source_id="mysql-replica-01",task="test"} for 1-th time, got value: '1' wait for valid metric of dm_syncer_replication_lag_sum{source_id="mysql-replica-01",task="test",worker="worker1"} for 1-th time, got value: '0' check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check ddl lag done! wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:13:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:13:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully check dml/skip lag done! wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:13:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:13:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 check zero job done! wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Mon May 6 17:13:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/metrics/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive make a dml job check diff successfully dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:13:50 CST 2024] <<<<<< test case metrics success! >>>>>> start running case: [new_collation_off] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. [Mon May 6 17:13:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:13:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_collation_off/conf/dm-task.yaml" check data check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:14:04 CST 2024] <<<<<< test case new_collation_off success! >>>>>> start running case: [only_dml] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:14:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:14:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/only_dml/source1.yaml" [Mon May 6 17:14:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/only_dml/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/conf/dm-task.yaml " check diff successfully PID of insert_data is 57709 dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000002 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000003 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000004 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000005 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000006 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000007 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000008 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000009 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 9-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000010 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000012 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000013 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000014 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000015 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000016 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000017 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000018 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000019 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 9-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000020 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000022 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000023 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000024 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000025 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000026 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000027 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000028 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000029 -s mysql-replica-01" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 9-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000030 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 1-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000031 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 2-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000033 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 3-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000034 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 4-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000035 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 5-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000036 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 6-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000037 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 7-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000038 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 8-th time, retry later dmctl test cmd: "purge-relay --filename dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000039 -s mysql-replica-02" earliest_relay_log1: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 earliest_relay_log2: dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001 purge relay log failed 9-th time, retry later check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/only_dml/run.sh: line 44: 57709 Terminated insert_data 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:14:57 CST 2024] <<<<<< test case only_dml success! >>>>>> start running case: [openapi] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:14:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:15:00 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time wait for rpc addr 127.0.0.1:8361 alive the 2-th time rpc addr 127.0.0.1:8361 is alive [Mon May 6 17:15:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:15:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: RELAY ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) dmctl test cmd: "query-status -s mysql-01" got=2 expected=2 got=1 expected=1 ('enable_relay_failed resp=', {u'error_code': 46004, u'error_msg': u'[code=46004:class=scheduler:scope=internal:level=medium], Message: dm-worker with name [no-worker] not exists'}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 ('get_source_status_failed resp=', {u'error_code': 46008, u'error_msg': u'[code=46008:class=scheduler:scope=internal:level=medium], Message: source config with ID no-mysql not exists'}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 1, u'data': [{u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('disable_relay_failed resp=', {u'error_code': 46004, u'error_msg': u'[code=46004:class=scheduler:scope=internal:level=medium], Message: dm-worker with name [no-worker] not exists'}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 ('get_source_status_success_no_relay resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('list_source_with_status_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'task_name_list': [], u'flavor': u'mysql', u'password': u'******', u'port': 3306, u'status_list': [{u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'', u'relay_catch_up_master': False, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}]}) ('get_source_status_success resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('get_source_status_success_with_relay resp=', {u'total': 2, u'data': [{u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'stage': u'Running'}, u'worker_name': u'worker1', u'source_name': u'mysql-01'}, {u'relay_status': {u'relay_dir': u'b3cad847-0b85-11ef-8754-d6df5ec14d97.000001', u'relay_catch_up_master': True, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-2', u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 488)', u'relay_binlog_gtid': u'', u'stage': u'Running'}, u'worker_name': u'worker2', u'source_name': u'mysql-01'}]}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 got=1 expected=1 delete_source_success >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: RELAY SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: SOURCE ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source_failed resp=', {u'error_code': 10001, u'error_msg': u'[code=10001:class=database:scope=upstream:level=high], Message: database driver error, RawCause: dial tcp :0: connect: connection refused, Workaround: Please check the database connection and the database config in configuration file.'}) ('update_source1_without_password_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': None, u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('list_source_with_status_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'task_name_list': [], u'flavor': u'mysql', u'password': u'******', u'port': 3306, u'status_list': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}]}) dmctl test cmd: "query-status -s mysql-01" got=1 expected=1 ('get_source_schemas_and_tables_success schema_resp=', [u'information_schema', u'Upper_DB', u'Upper_DB1', u'Upper_Db_IGNORE', u'adjust_gtid', u'async_checkpoint_flush', u'binlog_parse', u'check-task', u'checkpoint_transaction', u'checktask', u'checktask1', u'dm_syncer', u'dm_syncer_do_db', u'dm_syncer_ignore_db', u'downstream_diff_index1', u'downstream_more_column1', u'drop_column_with_index', u'dup_event', u'expr_filter', u'extend_column1', u'fake_rotate_event', u'foreign_key', u'full/mode', u'full_mode', u'gbk', u'gbk3', u'gtid', u'ha_test', u'ignore_db', u'lightning_mode', u'load_task1', u'load_task2', u'load_task4', u'metrics', u'mysql', u'only_dml', u'openapi', u'performance_schema', u'sys']) ('get_source_schemas_and_tables_success table_resp=', [u't1']) ('get_source_schemas_and_tables_success schema_resp=', [u'information_schema', u'Upper_DB', u'Upper_DB1', u'Upper_Db_IGNORE', u'adjust_gtid', u'async_checkpoint_flush', u'binlog_parse', u'check-task', u'checkpoint_transaction', u'checktask', u'checktask1', u'db-name', u'dm_syncer', u'dm_syncer_do_db', u'dm_syncer_ignore_db', u'downstream_diff_index1', u'downstream_more_column1', u'drop_column_with_index', u'dup_event', u'expr_filter', u'extend_column1', u'fake_rotate_event', u'foreign_key', u'full/mode', u'full_mode', u'gbk', u'gbk3', u'gtid', u'ha_test', u'ignore_db', u'lightning_mode', u'load_task1', u'load_task2', u'load_task4', u'metrics', u'mysql', u'only_dml', u'openapi', u'performance_schema', u'sys']) ('get_source_schemas_and_tables_success table_resp=', [u't1']) delete_source_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('delete_source_failed msg=', {u'error_code': 46008, u'error_msg': u'[code=46008:class=scheduler:scope=internal:level=medium], Message: source config with ID mysql-01 not exists'}) ('list_source_with_reverse resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: SOURCE SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: SHARD TASK ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) ('create_task_failed resp=', {u'error': u'error in openapi3filter.RequestError: request body has an error: doesn\'t match the schema: Error at "/task/shard_mode": value is not one of the allowed values'}) ('create_shard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'binlog_filter_rule': {u'rule-2': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}, u'rule-1': {u'ignore_event': [u'delete']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'rule-1'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'rule-2'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-shard" got=2 expected=2 dmctl test cmd: "query-status test-shard" got=2 expected=2 check diff successfully ('get_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name not a task name not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_illegal_char_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name t-\xcb!s`t not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 6, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3103)', u'syncer_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-12', u'synced': False, u'total_tps': 0, u'unresolved_groups': [{u'synced': None, u'first_location': u'', u'unsynced': [u"this DM-worker doesn't receive any shard DDL of this group"], u'target': u'`openapi`.`t`', u'ddl_list': None}], u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 2344)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': [u'ALTER TABLE `openapi`.`t` ADD COLUMN `aaa` INT'], u'recent_tps': 0, u'total_events': 6, u'master_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-11', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 2511)', u'syncer_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-10', u'synced': False, u'total_tps': 0, u'unresolved_groups': [{u'synced': [u'`openapi`.`t`'], u'first_location': u'position: (dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 2384), gtid-set: b41b494a-0b85-11ef-9114-d6df5ec14d97:1-10', u'unsynced': None, u'target': u'`openapi`.`t`', u'ddl_list': [u'ALTER TABLE `openapi`.`t` ADD COLUMN `aaa` INT']}], u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 2307)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'binlog_filter_rule': {u'mysql-01-filter-rule-0': {u'ignore_event': [u'delete']}, u'mysql-02-filter-rule-0': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-01-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-02-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'binlog_filter_rule': {u'mysql-01-filter-rule-0': {u'ignore_event': [u'delete']}, u'mysql-02-filter-rule-0': {u'ignore_sql': [u'alter table openapi.t add column aaa int;']}}, u'name': u'test-shard', u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-02-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'binlog_filter_rule': [u'mysql-01-filter-rule-0'], u'target': {u'table': u't', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'import_threads': 16, u'data_dir': u'./exported_data', u'export_threads': 4, u'consistency': u'auto'}, u'source_conf': [{u'source_name': u'mysql-02'}, {u'source_name': u'mysql-01'}]}}]}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: SHARD TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: MULTI TASK ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}}) dmctl test cmd: "query-status test-1" got=2 expected=2 dmctl test cmd: "query-status test-1" got=2 expected=2 ('get_task_list_with_status resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3891)', u'syncer_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3891)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3395)', u'syncer_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3395)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-1'}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-2'}}) dmctl test cmd: "query-status test-2" got=2 expected=2 dmctl test cmd: "query-status test-2" got=2 expected=2 ('get_task_list_with_status resp=', {u'total': 2, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task1_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3891)', u'syncer_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3891)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-1', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3395)', u'syncer_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3395)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-1'}, {u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'task2_target_table', u'schema': u'openapi'}}], u'task_mode': u'all', u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'status_list': [{u'source_name': u'mysql-01', u'name': u'test-2', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3891)', u'syncer_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-19', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3891)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-2', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 0, u'master_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-15', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3395)', u'syncer_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-15', u'synced': True, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 3395)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}], u'name': u'test-2'}]}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: MULTI TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: NO SHARD TASK ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}}) dmctl test cmd: "query-status test-no-shard" got=2 expected=2 dmctl test cmd: "query-status test-no-shard" got=2 expected=2 check diff successfully ('get_task_status_failed resp=', {u'error_code': 46018, u'error_msg': u'[code=46018:class=scheduler:scope=internal:level=medium], Message: task with name not a task name not exist, Workaround: Please use `query-status` command to see tasks.'}) ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-23', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 4679)', u'syncer_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-22', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 4425)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 4279)', u'syncer_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-18', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 4005)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}]}) delete_source_with_force_success ('get_task_status_success resp=', {u'total': 1, u'data': [{u'source_name': u'mysql-02', u'name': u'test-no-shard', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-19', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 4279)', u'syncer_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-18', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 4005)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}]}) ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}]}) ('get_task_schema_success schema resp=', [u'openapi']) ('get_task_schema_success table resp=', [u't2']) ('get_task_schema_success create table resp=', {u'table_name': u't2', u'schema_create_sql': u'CREATE TABLE `t2` ( `i` tinyint(4) DEFAULT NULL, `j` int(11) DEFAULT NULL, UNIQUE KEY `j` (`j`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin', u'schema_name': u'openapi'}) ('get_task_schema_success table resp=', [u't2']) ('delete_task_failed resp=', {u'error_code': 49001, u'error_msg': u'[code=49001:class=openapi:scope=internal:level=high], Message: task test-no-shard have running subtasks, please stop them or delete task with force.'}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: NO SHARD TASK SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: TASK TEMPLATES ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_task_template_failed resp=', {u'error': u'error in openapi3filter.RequestError: request body has an error: doesn\'t match the schema: Error at "/shard_mode": value is not one of the allowed values'}) ('create_task_template_success resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}) ('list_task_template resp=', {u'total': 1, u'data': [{u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}]}) ('get_task_template resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'name': u'test-1'}) ('update_task_template_success resp=', {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'shard_mode': u'pessimistic', u'task_mode': u'full', u'name': u'test-1'}) delete_task_template ('list_task_template resp=', {u'total': 0, u'data': []}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}}) dmctl test cmd: "query-status test-1" got=2 expected=2 dmctl test cmd: "query-status test-1" got=2 expected=2 ('import_task_template resp=', {u'success_task_list': [u'test-1'], u'failed_task_list': []}) ('list_task_template resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-1'}]}) dmctl test cmd: "config task test-1 --path /tmp/dm_test/openapi/get_task_from_task.yaml" dmctl test cmd: "config task-template test-1 --path /tmp/dm_test/openapi/get_task_from_task_template.yaml" delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: TASK TEMPLATES >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: NO SHARD TASK DUMP STATUS wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:15:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:15:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard-dump-status'}}) dmctl test cmd: "query-status test-no-shard-dump-status" got=2 expected=2 dmctl test cmd: "query-status test-no-shard-dump-status" got=2 expected=2 got=2 expected=2 ('check_dump_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-02', u'name': u'test-no-shard-dump-status', u'unresolved_ddl_lock_id': u'', u'worker_name': u'worker2', u'dump_status': {u'estimate_total_rows': 0, u'finished_bytes': 0, u'completed_tables': 0, u'total_tables': 0, u'finished_rows': 0}, u'unit': u'Dump', u'stage': u'Running'}, {u'source_name': u'mysql-01', u'name': u'test-no-shard-dump-status', u'unresolved_ddl_lock_id': u'', u'worker_name': u'worker1', u'dump_status': {u'estimate_total_rows': 0, u'finished_bytes': 0, u'completed_tables': 0, u'total_tables': 0, u'finished_rows': 0}, u'unit': u'Dump', u'stage': u'Running'}]}) wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit ('get_task_status_success_but_worker_meet_error resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-no-shard-dump-status', u'worker_name': u'', u'error_msg': u'[code=38029:class=dm-master:scope=internal:level=high], Message: mysql-01 relevant worker-client not found, Workaround: Please use list-member command to see if the some workers are offline.', u'unit': u'', u'stage': u''}, {u'source_name': u'mysql-02', u'name': u'test-no-shard-dump-status', u'worker_name': u'', u'error_msg': u'[code=38029:class=dm-master:scope=internal:level=high], Message: mysql-02 relevant worker-client not found, Workaround: Please use list-member command to see if the some workers are offline.', u'unit': u'', u'stage': u''}]}) Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-worker.test already exit [Mon May 6 17:15:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:15:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: NO SHARD TASK DUMP STATUS SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: COMPLEX OPERATION ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-complex'}}) dmctl test cmd: "query-status test-complex" got=2 expected=2 ('get_task_list resp=', {u'total': 1, u'data': [{u'strict_optimistic_shard_mode': False, u'source_config': {u'incr_migrate_conf': {u'repl_batch': 100, u'repl_threads': 16}, u'full_migrate_conf': {u'data_dir': u'./dumped_data', u'export_threads': 4, u'import_threads': 16}, u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'replace', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-complex'}]}) dmctl test cmd: "query-status test-complex" got=2 expected=2 check diff successfully ('get_task_status_success resp=', {u'total': 2, u'data': [{u'source_name': u'mysql-01', u'name': u'test-complex', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-33', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 6589)', u'syncer_binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-32', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 6335)'}, u'worker_name': u'worker1', u'unit': u'Sync', u'stage': u'Running'}, {u'source_name': u'mysql-02', u'name': u'test-complex', u'unresolved_ddl_lock_id': u'', u'sync_status': {u'binlog_type': u'remote', u'blocking_ddls': None, u'recent_tps': 0, u'total_events': 5, u'master_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-29', u'seconds_behind_master': 0, u'master_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 6441)', u'syncer_binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-28', u'synced': False, u'total_tps': 0, u'unresolved_groups': None, u'syncer_binlog': u'(dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001, 6167)'}, u'worker_name': u'worker2', u'unit': u'Sync', u'stage': u'Running'}]}) check diff successfully delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: COMPLEX OPERATION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: TEST TASK WITH IGNORE CHECK ITEMS ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) ('create_task_with_precheck resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'name': u'test-no-ignore-no-error', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u''], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-no-ignore-no-error" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_task_with_precheck resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 21,\n\t\t\t\t"name": "sharding table `openapi`.`t` consistency checking",\n\t\t\t\t"desc": "check consistency of sharding table structures",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "sourceID mysql-01 table {openapi t} of sharding `openapi`.`t` have auto-increment key, please make sure them don\'t conflict in target table!"\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "sourceID mysql-02 table {openapi t} of sharding `openapi`.`t` have auto-increment key, please make sure them don\'t conflict in target table!"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "If happen conflict, please handle it by yourself. You can refer to https://docs.pingcap.com/tidb-data-migration/stable/shard-merge-best-practices/#handle-conflicts-between-primary-keys-or-unique-indexes-across-multiple-sharded-tables",\n\t\t\t\t"extra": "auto-increment key checking"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 22,\n\t\t\t"successful": 21,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'name': u'test-no-ignore-has-warn', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u''], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-no-ignore-has-warn" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_task_with_precheck resp=', {u'error_code': 26005, u'error_msg': u'[code=26005:class=dm-master:scope=internal:level=medium], Message: fail to check synchronization configuration with type: check was failed, please see detail\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 21,\n\t\t\t\t"name": "sharding table `openapi`.`t` consistency checking",\n\t\t\t\t"desc": "check consistency of sharding table structures",\n\t\t\t\t"state": "fail",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "fail",\n\t\t\t\t\t\t"short_error": "column length mismatch (self: 3 vs other: 2)",\n\t\t\t\t\t\t"self": "sourceID mysql-01 table {openapi t} columns [id i j]",\n\t\t\t\t\t\t"other": "sourceID mysql-02 table `openapi`.`t` columns [i j]"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "please set same table structure for sharding tables",\n\t\t\t\t"extra": "error on sharding `openapi`.`t`"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": false,\n\t\t\t"total": 22,\n\t\t\t"successful": 21,\n\t\t\t"failed": 1,\n\t\t\t"warning": 0\n\t\t}\n\t}'}) dmctl test cmd: "query-status test-no-ignore-has-error" got=1 expected=1 ('create_task_with_precheck resp=', {u'check_result': u'pre-check is passed. ', u'task': {u'name': u'test-has-ignore-without-error', u'table_migrate_rule': [{u'source': {u'table': u't*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}, {u'source': {u'table': u't*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u't', u'schema': u'openapi'}}], u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'ignore_checking_items': [u'version', u'schema_of_shard_tables'], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'shard_mode': u'pessimistic', u'task_mode': u'all', u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}}}) dmctl test cmd: "query-status test-has-ignore-without-error" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: TEST TASK WITH IGNORE CHECK ITEMS SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: DELETE TASK WITH STOPPED DOWNSTREAM ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test-no-shard'}}) dmctl test cmd: "query-status test-no-shard" got=2 expected=2 tidb_server_num tidb-server alive wait process tidb-server exit... process tidb-server already exit ('delete_task_failed resp=', {u'error_code': 10001, u'error_msg': u'[code=10001:class=database:scope=downstream:level=high], Message: database driver error, RawCause: dial tcp 127.0.0.1:4000: connect: connection refused, Workaround: Please check the database connection and the database config in configuration file.'}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: DELETE TASK WITH STOPPED DOWNSTREAM SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: START TASK WITH CONDITION ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_no_source_meta'}}) dmctl test cmd: "query-status incremental_task_no_source_meta" got=2 expected=2 ('start_task_failed resp=', {u'error_code': 20022, u'error_msg': u'[code=20022:class=config:scope=internal:level=medium], Message: mysql-instance(mysql-01) must set meta for task-mode incremental, Workaround: Please check the `meta` config in task configuration file.'}) delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 10,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 20,\n\t\t\t"successful": 19,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01', u'binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-58', u'binlog_name': u'dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001', u'binlog_pos': 11460}, {u'source_name': u'mysql-02', u'binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-54', u'binlog_name': u'dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001', u'binlog_pos': 11906}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_gtid'}}) dmctl test cmd: "query-status incremental_task_use_gtid" got=2 expected=2 dmctl test cmd: "query-status incremental_task_use_gtid" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_start_time'}}) dmctl test cmd: "query-status incremental_task_use_start_time" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_use_start_time" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_use_start_time_after_create'}}) dmctl test cmd: "query-status incremental_task_use_start_time_after_create" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_use_start_time_after_create" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 10,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 20,\n\t\t\t"successful": 19,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01', u'binlog_gtid': u'b3cad847-0b85-11ef-8754-d6df5ec14d97:1-70', u'binlog_name': u'dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001', u'binlog_pos': 13696}, {u'source_name': u'mysql-02', u'binlog_gtid': u'b41b494a-0b85-11ef-9114-d6df5ec14d97:1-66', u'binlog_name': u'dm-it-dd48039b-fa36-4d08-b528-750e7cb39fb8-pk9vm-8kvx1-bin.000001', u'binlog_pos': 14442}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_both_gtid_start_time'}}) dmctl test cmd: "query-status incremental_task_both_gtid_start_time" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_both_gtid_start_time" got=2 expected=2 delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:16:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:16:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('create_incremental_task_with_gtid_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 9,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 18,\n\t\t\t"successful": 17,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm_meta', u'task_mode': u'incremental', u'name': u'incremental_task_no_duration_but_error'}}) dmctl test cmd: "query-status incremental_task_no_duration_but_error" got=2 expected=2 start_task_with_condition success dmctl test cmd: "query-status incremental_task_no_duration_but_error" got=0 expected=2 command: query-status incremental_task_no_duration_but_error Duplicate entry count: 0 != expected: 2, failed the 0-th time, will retry again got=2 expected=2 start_task_with_condition success run tidb sql failed 1-th time, retry later delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: STOP TASK WITH CONDITION ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_by_openapi_success resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker1', u'source_name': u'mysql-01'}]}) ('create_source1_success resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3307}) ('list_source_by_openapi_success resp=', {u'total': 2, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}, {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-02', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3307}]}) ('get_source_status_success resp=', {u'total': 1, u'data': [{u'worker_name': u'worker2', u'source_name': u'mysql-02'}]}) wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:16:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:16:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('create_noshard_task_success resp=', {u'check_result': u'fail to check synchronization configuration with type: no errors but some warnings\n detail: {\n\t\t"results": [\n\t\t\t{\n\t\t\t\t"id": 13,\n\t\t\t\t"name": "mysql_version",\n\t\t\t\t"desc": "check whether mysql version is satisfied",\n\t\t\t\t"state": "warn",\n\t\t\t\t"errors": [\n\t\t\t\t\t{\n\t\t\t\t\t\t"severity": "warn",\n\t\t\t\t\t\t"short_error": "version suggested earlier than 8.0.0 but got 8.0.21"\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\t"instruction": "It is recommended that you select a database version that meets the requirements before performing data migration. Otherwise data inconsistency or task exceptions might occur.",\n\t\t\t\t"extra": "address of db instance - 127.0.0.1:3307"\n\t\t\t}\n\t\t],\n\t\t"summary": {\n\t\t\t"passed": true,\n\t\t\t"total": 23,\n\t\t\t"successful": 22,\n\t\t\t"failed": 0,\n\t\t\t"warning": 1\n\t\t}\n\t}', u'task': {u'source_config': {u'source_conf': [{u'source_name': u'mysql-01'}, {u'source_name': u'mysql-02'}]}, u'enhance_online_schema_change': True, u'on_duplicate': u'error', u'table_migrate_rule': [{u'source': {u'table': u'*', u'source_name': u'mysql-01', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}, {u'source': {u'table': u'*', u'source_name': u'mysql-02', u'schema': u'openapi'}, u'target': {u'table': u'', u'schema': u'openapi'}}], u'target_config': {u'user': u'root', u'host': u'127.0.0.1', u'password': u'', u'port': 4000, u'security': None}, u'meta_schema': u'dm-meta', u'task_mode': u'all', u'name': u'test_wait_time_on_stop'}}) dmctl test cmd: "query-status test_wait_time_on_stop" got=2 expected=2 dmctl test cmd: "query-status test_wait_time_on_stop" got=2 expected=2 check diff successfully error check process dm-worker.test already exit [Mon May 6 17:16:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:16:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive delete_task_success ('get_task_list resp=', {u'total': 0, u'data': []}) delete_source_with_force_success delete_source_with_force_success ('list_source_by_openapi_success resp=', {u'total': 0, u'data': []}) ('get_task_list resp=', {u'total': 0, u'data': []}) >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: START TASK WITH CONDITION SUCCESS >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: REVERSE HTTPS 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:16:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/openapi/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:16:39 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/openapi/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time wait for rpc addr 127.0.0.1:8361 alive the 2-th time rpc addr 127.0.0.1:8361 is alive [Mon May 6 17:16:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/openapi/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:16:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/openapi/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive ('create_source_success_https resp=', {u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'host': u'127.0.0.1', u'user': u'root', u'security': None, u'password': u'123456', u'port': 3306}) ('list_source_success_https resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) ('list_source_with_reverse_https resp=', {u'total': 1, u'data': [{u'enable_gtid': False, u'enable': True, u'source_name': u'mysql-01', u'security': None, u'relay_config': {u'relay_binlog_name': u'', u'relay_dir': u'relay-dir', u'relay_binlog_gtid': u'', u'enable_relay': False}, u'purge': {u'remain_space': 15, u'expires': 0, u'interval': 3600}, u'host': u'127.0.0.1', u'user': u'root', u'flavor': u'mysql', u'password': u'******', u'port': 3306}]}) 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:16:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:16:57 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time wait for rpc addr 127.0.0.1:8361 alive the 2-th time rpc addr 127.0.0.1:8361 is alive [Mon May 6 17:17:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:17:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/openapi/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: REVERSE HTTPS ('list_master_success resp=', {u'total': 2, u'data': [{u'name': u'master1', u'addr': u'http://127.0.0.1:8291', u'alive': True, u'leader': True}, {u'name': u'master2', u'addr': u'http://127.0.0.1:8292', u'alive': True, u'leader': False}]}) ('list_worker_success resp=', {u'total': 2, u'data': [{u'bound_source_name': u'', u'bound_stage': u'free', u'addr': u'127.0.0.1:8262', u'name': u'worker1'}, {u'bound_source_name': u'', u'bound_stage': u'free', u'addr': u'127.0.0.1:8263', u'name': u'worker2'}]}) ('delete_master_failed resp=', {u'error_code': 0, u'error_msg': u'etcdserver: unhealthy cluster'}, 'retry cnt=', 0) delete_master_with_retry_success ('list_master_success resp=', {u'total': 1, u'data': [{u'name': u'master1', u'addr': u'http://127.0.0.1:8291', u'alive': True, u'leader': True}]}) ('delete_worker_failed resp=', {u'error_code': 46005, u'error_msg': u'[code=46005:class=scheduler:scope=internal:level=medium], Message: dm-worker with name worker1 is still online, Workaround: Please shut it down first.'}) wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit delete_worker_with_retry_success ('list_worker_success resp=', {u'total': 1, u'data': [{u'bound_source_name': u'', u'bound_stage': u'offline', u'addr': u'127.0.0.1:8263', u'name': u'worker2'}]}) 2 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:17:23 CST 2024] <<<<<< test case openapi success! >>>>>> start running case: [s3_dumpling_lightning] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit Starting PD... Release Version: v7.5.1-6-g78f4254e3 Edition: Community Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815 Git Branch: release-7.5 UTC Build Time: 2024-04-30 02:49:46 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:2379; Connection refused 2024-05-06 17:17:23.656146 W | pkg/fileutil: check file permission: directory "/tmp/dm_test/s3_dumpling_lightning.downstream/pd" exist, but the permission is "drwxr-xr-x". The recommended permission is "-rwx------" to prevent possible unprivileged access to the data. % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 10 100 10 0 0 18348 0 --:--:-- --:--:-- --:--:-- 10000 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 105k 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 117 100 117 0 0 98319 0 --:--:-- --:--:-- --:--:-- 114k {"connections":0,"version":"8.0.11-TiDB-v7.5.1-48-g066b97f741","git_hash":"066b97f741e6fbef0740e073940cea08d13b3f43"}process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDBA4FB299EC7 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 09:17:39 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-master.test already exit Usage: kill [options] [...] Options: -a, --all do not restrict the name-to-pid conversion to processes with the same uid as the present process -s, --signal send specified signal -q, --queue use sigqueue(2) rather than kill(2) -p, --pid print pids without signaling them -l, --list [=] list signal names, or convert one to a name -L, --table list signal names and numbers -h, --help display this help and exit -V, --version output version information and exit For more details see kill(1). process dm-worker.test already exit [Mon May 6 17:17:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:17:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:17:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 4-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 66651 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDBAA80A8295B < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 09:18:03 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:18:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:18:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:18:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 67302 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test without check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDBB1713D8339 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 09:18:33 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:18:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:18:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:18:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 67974 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with special task-name and check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDBB82B5DC159 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 09:19:02 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:19:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:19:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:19:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml --remove-meta" check task result run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later run tidb sql failed 3-th time, retry later wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 68635 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test with special task-name and without check dump files success process minio already exit * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8688; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:8688 Browser Access: http://127.0.0.1:8688 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 8688 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8688 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8688 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCDBBF1F3CF013 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 09:19:32 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Mon May 6 17:19:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:19:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:19:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/s3_dumpling_lightning/source2.yaml" prepare source data start task dmctl test cmd: "start-task /tmp/dm_test/s3_dumpling_lightning/dm-task.yaml" error check wait process minio exit... process minio already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/s3_dumpling_lightning/run.sh: line 49: 69308 Killed bin/minio server --address $S3_ENDPOINT "$s3_DBPATH" run s3 test error check success Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 179 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:19:58 CST 2024] <<<<<< test case s3_dumpling_lightning success! >>>>>> start running case: [sequence_sharding_optimistic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:19:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:19:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:20:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 341 100 216 100 125 65021 37627 --:--:-- --:--:-- --:--:-- 72000 dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_optimistic/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_optimistic/source2.yaml" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 407 100 282 100 125 60179 26675 --:--:-- --:--:-- --:--:-- 70500 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-task.yaml --remove-meta" check diff successfully ERROR 1146 (42S02) at line 1: Table 'sharding_seq_tmp.t1' doesn't exist run tidb sql failed 1-th time, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 398 100 273 100 125 70560 32308 --:--:-- --:--:-- --:--:-- 91000 dmctl test cmd: "pause-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=2 expected=2 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 512 100 387 100 125 103k 34218 --:--:-- --:--:-- --:--:-- 125k dmctl test cmd: "resume-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=3 expected=3 restart dm-worker1 wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit [Mon May 6 17:20:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_optimistic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "shard-ddl-lock unlock non-exist-task-`test_db`.`test_table`" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" dmctl test cmd: "query-status sequence_sharding_optimistic" got=1 expected=1 dmctl test cmd: "resume-task sequence_sharding_optimistic" got=3 expected=3 check diff successfully dmctl test cmd: "pause-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=2 expected=2 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 508 100 383 100 125 87562 28577 --:--:-- --:--:-- --:--:-- 95750 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 291 100 166 100 125 53808 40518 --:--:-- --:--:-- --:--:-- 55333 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 538 100 370 100 168 71774 32589 --:--:-- --:--:-- --:--:-- 74000 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 508 100 383 100 125 117k 39258 --:--:-- --:--:-- --:--:-- 124k dmctl test cmd: "binlog-schema list -s mysql-replica-01,mysql-replica-02 sequence_sharding_optimistic sharding_seq_opt t2" dmctl test cmd: "binlog-schema delete -s mysql-replica-01 sequence_sharding_optimistic sharding_seq_opt t2" dmctl test cmd: "binlog-schema update -s mysql-replica-01 sequence_sharding_optimistic sharding_seq_opt t1 /tmp/dm_test/sequence_sharding_optimistic/schema.sql" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 508 100 383 100 125 98k 33033 --:--:-- --:--:-- --:--:-- 124k { "result": true, "msg": "", "sources": [ { "result": true, "msg": "CREATE TABLE `t1` ( `id` bigint(20) NOT NULL, `c2` varchar(20) DEFAULT NULL, `c3` bigint(11) DEFAULT NULL, PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin", "source": "mysql-replica-01", "worker": "worker1" } ] }dmctl test cmd: "resume-task sequence_sharding_optimistic" got=3 expected=3 dmctl test cmd: "query-status sequence_sharding_optimistic" got=3 expected=3 check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:20:18 CST 2024] <<<<<< test case sequence_sharding_optimistic success! >>>>>> start running case: [sequence_sharding_removemeta] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:20:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:20:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:20:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_removemeta/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding_removemeta/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-task.yaml " check diff successfully dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status sequence_sharding_removemeta" got=0 expected=0 got=0 expected=0 dmctl test cmd: "stop-task sequence_sharding_removemeta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding_removemeta/conf/dm-task.yaml --remove-meta" dmctl test cmd: "shard-ddl-lock" got=1 expected=1 check diff successfully dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status sequence_sharding_removemeta" got=1 expected=1 dmctl test cmd: "stop-task sequence_sharding_removemeta" dmctl test cmd: "shard-ddl-lock unlock sequence_sharding_removemeta-`sharding_target3`.`t_target`" dmctl test cmd: "shard-ddl-lock" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:20:50 CST 2024] <<<<<< test case sequence_sharding_removemeta success! >>>>>> start running case: [shardddl_optimistic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:20:51 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:20:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl_optimistic/source1.yaml" [Mon May 6 17:20:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl_optimistic/source2.yaml" [Mon May 6 17:20:56 CST 2024] <<<<<< start DM-DIFFERENT_SCHEMA_FULL optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:20:59 CST 2024] <<<<<< finish DM-DIFFERENT_SCHEMA_FULL optimistic >>>>>> [Mon May 6 17:20:59 CST 2024] <<<<<< start DM-DIFFERENT_SCHEMA_INCREMENTAL optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "pause-task test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 245 100 245 0 0 62917 0 --:--:-- --:--:-- --:--:-- 81666 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 275 100 275 0 0 110k 0 --:--:-- --:--:-- --:--:-- 134k % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 268 100 268 0 0 92254 0 --:--:-- --:--:-- --:--:-- 130k % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 298 100 298 0 0 122k 0 --:--:-- --:--:-- --:--:-- 145k dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/shardddl_optimistic/task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d shardddl1 -t tb1 /tmp/dm_test/shardddl_optimistic/schema11.sql" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d shardddl1 -t tb2 /tmp/dm_test/shardddl_optimistic/schema12.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d shardddl1 -t tb1 /tmp/dm_test/shardddl_optimistic/schema21.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d shardddl1 -t tb2 /tmp/dm_test/shardddl_optimistic/schema22.sql" dmctl test cmd: "resume-task test" run tidb sql failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:21:05 CST 2024] <<<<<< finish DM-DIFFERENT_SCHEMA_INCREMENTAL optimistic >>>>>> [Mon May 6 17:21:05 CST 2024] <<<<<< start DM-RESTART_TASK_MASTER_WORKER optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later restart worker2 restart dm-worker2 wait process worker2 exit... wait process worker2 exit... process worker2 already exit [Mon May 6 17:21:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later restart master restart dm-master wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Mon May 6 17:21:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later restart task restart task dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later restart task restart task dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:21:29 CST 2024] <<<<<< finish DM-RESTART_TASK_MASTER_WORKER optimistic >>>>>> [Mon May 6 17:21:29 CST 2024] <<<<<< start DM-STOP_TASK_FOR_A_SOURCE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test -s mysql-replica-02" run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml -s mysql-replica-02" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:21:36 CST 2024] <<<<<< finish DM-STOP_TASK_FOR_A_SOURCE optimistic >>>>>> [Mon May 6 17:21:36 CST 2024] <<<<<< start DM-UPDATE_BA_ROUTE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/shardddl_optimistic/task.yaml" dmctl test cmd: "show-ddl-locks" got=1 expected=1 run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Mon May 6 17:21:46 CST 2024] <<<<<< finish DM-UPDATE_BA_ROUTE optimistic >>>>>> [Mon May 6 17:21:46 CST 2024] <<<<<< start DM-CREATE_DROP_TABLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl_optimistic/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later run tidb sql failed 1-th time, retry later dmctl test cmd: "show-ddl-locks" dmctl test cmd: "stop-task test" [Mon May 6 17:21:52 CST 2024] <<<<<< finish DM-CREATE_DROP_TABLE optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:21:57 CST 2024] <<<<<< test case shardddl_optimistic success! >>>>>> start running case: [slow_relay_writer] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:21:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:21:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/slow_relay_writer/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/slow_relay_writer/conf/dm-task.yaml " check diff successfully start incremental_data finish incremental_data dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:22:17 CST 2024] <<<<<< test case slow_relay_writer success! >>>>>> start running case: [sql_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:22:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:22:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:22:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sql_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sql_mode/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sql_mode/conf/dm-task.yaml " check diff successfully check diff successfully check diff successfully check diff successfully check diff successfully ERROR 1146 (42S02) at line 1: Table 'sql_mode.t0' doesn't exist run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:22:31 CST 2024] <<<<<< test case sql_mode success! >>>>>> start running case: [sync_collation] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:22:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:22:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:22:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sync_collation/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sync_collation/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/sync_collation/source2.yaml" prepare data start task dmctl test cmd: "start-task /tmp/dm_test/sync_collation/dm-task.yaml --remove-meta" check full phase prepare incremental data check incremental phase run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task sync_collation" prepare data for full phase error test dmctl test cmd: "start-task /tmp/dm_test/sync_collation/dm-task.yaml --remove-meta" check full phase error dmctl test cmd: "query-status sync_collation" got=1 expected=1 dmctl test cmd: "stop-task sync_collation" prepare data for incremental phase error test dmctl test cmd: "start-task /tmp/dm_test/sync_collation/dm-task.yaml --remove-meta" check incremental phase error dmctl test cmd: "query-status sync_collation" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:22:48 CST 2024] <<<<<< test case sync_collation success! >>>>>> start running case: [tracker_ignored_ddl] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:22:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:22:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tracker_ignored_ddl/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-task.yaml " check diff successfully increment1 check success dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "resume-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 increment2 check success dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/tracker_ignored_ddl/source1.yaml" [Mon May 6 17:22:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-master.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:22:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-worker.toml >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tracker_ignored_ddl/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tracker_ignored_ddl/conf/dm-task.yaml " check diff successfully increment1 check success dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "resume-task test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 increment2 check success dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/tracker_ignored_ddl/source1.yaml" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:23:07 CST 2024] <<<<<< test case tracker_ignored_ddl success! >>>>>> start running case: [validator_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh... Verbose mode = false --> full mode, check we validate different data types(gtid=false, relay=false) 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:23:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:23:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:23:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=false, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:23:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:23:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:23:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:23:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:23:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:23:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:23:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:23:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:23:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> fast mode, check we validate different data types 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:24:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:24:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:24:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: full mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:24:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:24:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:24:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: fast mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:24:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:24:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:24:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check update pk(split into insert and delete) dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 7\/1\/2" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator panic and we can catch it 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:24:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:24:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:24:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test validator panic count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 --> check validator worker panic and we can catch it 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:24:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:25:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:25:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator stop when pending row size too large 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:25:16 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:25:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:25:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1+ got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator stop when pending row count too many 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:25:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:25:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:25:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1+ got=1 expected=1 got=1 expected=1 got=1 expected=1 --> start validator on the fly, validate from current syncer progress(gtid=false, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:25:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:25:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:25:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=false, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:26:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:26:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:26:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=true, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:26:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:26:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:26:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator on the fly, validate from current syncer progress(gtid=true, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:26:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:26:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:26:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time < min mysql binlog pos(gtid=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:27:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:27:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:27:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time < min mysql binlog pos(gtid=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:27:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:27:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:27:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time which is in range of mysql binlog(gtid=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:27:42 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:27:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:27:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2024-05-06 11:27:51' test" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time which is in range of mysql binlog(gtid=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:28:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:28:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:28:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2024-05-06 11:28:12' test" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 check diff successfully --> start validator from time > max mysql binlog pos 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:28:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:28:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:28:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2026-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> single varchar col pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:28:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:28:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:28:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> single datetime col pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:28:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:28:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:28:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> compound pk (datetime, timestamp, int, varchar) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:29:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:29:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:29:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/4\/3" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table without primary key 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:29:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:29:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:29:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=1 command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table is deleted on downstream 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:29:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:29:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:29:41 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> table in schema-tracker has less column than binlog 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:29:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:29:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:29:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> pk column of downstream table not in range of binlog column 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:30:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:30:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:30:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> stopped validator fail over 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:30:16 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:30:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:30:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 run tidb sql failed 1-th time, retry later dmctl test cmd: "validation stop test" got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process worker1 exit... wait process worker1 exit... process worker1 already exit dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation start test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/3\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> filter online ddl shadow table 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:30:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:30:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:30:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/test-filter.yaml --remove-meta" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> filter by ba list dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> filter by filter-rules dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 7\/2\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=3 expected=3 --> validate when syncer is stopped 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:31:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:31:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:31:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=1 expected=1 got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again got=1 expected=1 got=1 expected=1 --> test duplicate auto-incr pk 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:33:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:33:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:33:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/sharding-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=2 command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 1 != expected: 2, failed the 0-th time, will retry again got=2 expected=2 got=2 expected=2 got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:33:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:33:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Mon May 6 17:33:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-long-interval.yaml --remove-meta" dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/1" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-gtid b3cad847-0b85-11ef-8754-d6df5ec14d97:1-79" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=0 expected=1 command: query-status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-gtid b3cad847-0b85-11ef-8754-d6df5ec14d97:1-81" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:33:55 CST 2024] <<<<<< test case validator_basic success! >>>>>> restore config restore time_zone start running case: [validator_checkpoint] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh... Verbose mode = false --> check persist checkpoint and data with 2 source 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:33:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:33:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:33:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta" dmctl test cmd: ""unit": "Sync"" dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test processedRowsStatus": "insert\/update\/delete: 3\/1\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 --> check validator can restart from previous position on fail over restart dm-worker1 wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Mon May 6 17:34:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive restart dm-worker2 wait process worker2 exit... wait process worker2 exit... process worker2 already exit [Mon May 6 17:34:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later --> check validator persist fail in the middle 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:34:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:34:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Mon May 6 17:34:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta" dmctl test cmd: ""unit": "Sync"" dmctl test cmd: "validation status test" got=0 expected=2 command: validation status test "stage": "Stopped" count: 0 != expected: 2, failed the 0-th time, will retry again got=2 expected=2 got=2 expected=2 got=2 expected=2 got=2 expected=2 got=2 expected=2 --> after resume and persist again, meta data should be consistent dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml " dmctl test cmd: "validation status test" got=4 expected=4 got=2 expected=2 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later --> check validate success after insert data manually dmctl test cmd: "validation status test" got=4 expected=4 got=2 expected=2 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again got=4 expected=4 got=2 expected=2 got=2 expected=2 got=2 expected=2 run tidb sql failed 1-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:34:45 CST 2024] <<<<<< test case validator_checkpoint success! >>>>>> start running case: [incompatible_ddl_changes] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:34:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:34:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test event reorganize table partition count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "stop-task test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:35:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Mon May 6 17:35:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incompatible_ddl_changes/conf/dm-task1.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test event reorganize table partition count: 0 != expected: 1, failed the 0-th time, will retry again got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 dmctl test cmd: "stop-task test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Mon May 6 17:35:30 CST 2024] <<<<<< test case incompatible_ddl_changes success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-307/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS