Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/release-7.5/pull_dm_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t Agent pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t is provisioned from template pingcap_tiflow_release-7_5_pull_dm_integration_test_545-mw8nr-mlmn6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7f42cba2238e878b4b88c43906632fc14f6d40ce" jenkins/label: "pingcap_tiflow_release-7_5_pull_dm_integration_test_545-mw8nr" name: "pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t" - name: "JENKINS_NAME" value: "pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git rev-list --no-walk e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=db7bd23a-8416-49cf-bc70-4391073ce22e JENKINS_NODE_COOKIE=9e1478f6-5970-4028-98e8-1b193eb2711a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/ GOLANG_VERSION=1.21.10 HOSTNAME=pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_dm_integration_test","buildid":"1796082933635747840","prowjobid":"db7bd23a-8416-49cf-bc70-4391073ce22e","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"release-7.5","base_sha":"26b901cb5b23f8c42b05e5a00514b52bef2efe24","base_link":"https://github.com/pingcap/tiflow/commit/26b901cb5b23f8c42b05e5a00514b52bef2efe24","pulls":[{"number":11200,"author":"ti-chi-bot","sha":"8f92142d7874bfad2a8db6ca100604c383fca9df","title":"*: copy seahash from blainsmith.com/go/seahash (#11193)","link":"https://github.com/pingcap/tiflow/pull/11200","commit_link":"https://github.com/pingcap/tiflow/pull/11200/commits/8f92142d7874bfad2a8db6ca100604c383fca9df","author_link":"https://github.com/ti-chi-bot"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/release-7.5/pull_dm_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#545 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796082933635747840 GOLANG_DOWNLOAD_SHA256=e330e5d977bf4f3bdc157bc46cf41afa5b13d66c914e12fd6b694ccda65fcf92 JOB_BASE_NAME=pull_dm_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=44e9fdd1b388d71f07c1fc94e6fccf31c512143e RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_release-7_5_pull_dm_integration_test_545-mw8nr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-24341cbb37398a3d76ef457a002bbb77ebf3752146c282247622af04dd7ac429 NODE_LABELS=pingcap_tiflow_release-7_5_pull_dm_integration_test_545-mw8nr pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/display/redirect BUILD_NUMBER=545 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.10.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.10' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build586132230=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t -c golang -- bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-release-7-5-pull-dm-integration-test-545-m-zmb2t -c golang -- bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 38580 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 9318d46a298e58ca (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Thu May 30 07:35:41 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11200/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/kv/shared_client.go, go.mod, go.sum, pkg/spanz/hash_map.go, pkg/util/seahash/LICENSE.md, pkg/util/seahash/seahash.go, pkg/util/seahash/seahash_test.go, scripts/check-copyright.sh] [Pipeline] echo diff file matched: go.mod [Pipeline] echo matched, some diff files full path start with dm/ or pkg/ or go.mod, run the dm integration test [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-26b901c-8f92142) 208203776 bytes in 7.57 secs (27503824 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/.git/ .git HEAD is now at 8f92142d7 fix POST git-upload-pack (686 bytes) From https://github.com/pingcap/tiflow = [up to date] release-7.5 -> origin/release-7.5 = [up to date] refs/pull/11200/head -> origin/pr/11200/head Previous HEAD position was 8f92142d7 fix HEAD is now at 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) ๐Ÿšง Checkouting to base SHA:26b901cb5b23f8c42b05e5a00514b52bef2efe24... HEAD is now at 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 26b901cb5b23f8c42b05e5a00514b52bef2efe24 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) bf93d7d63 dm: fix update dml loss if binary column is primary key (#10685) (#10691) 39a17af6e dep(*): move diff package to the tiflow and get rid of the tidb-tools (#11164) (#11168) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 26b901cb5b23f8c42b05e5a00514b52bef2efe24 ... Updating 26b901cb5..8f92142d7 Fast-forward cdc/kv/shared_client.go | 2 +- go.mod | 10 --- go.sum | 34 --------- pkg/spanz/hash_map.go | 2 +- pkg/util/seahash/LICENSE.md | 9 +++ pkg/util/seahash/seahash.go | 159 +++++++++++++++++++++++++++++++++++++++ pkg/util/seahash/seahash_test.go | 75 ++++++++++++++++++ scripts/check-copyright.sh | 2 +- 8 files changed, 246 insertions(+), 47 deletions(-) create mode 100644 pkg/util/seahash/LICENSE.md create mode 100644 pkg/util/seahash/seahash.go create mode 100644 pkg/util/seahash/seahash_test.go ๐Ÿงพ Pre-merged result: 8f92142d7874bfad2a8db6ca100604c383fca9df 8f92142d7 fix 645f6fc14 This is an automated cherry-pick of #11193 26b901cb5 dm: upgrade go-mysql library (#11043) (#11189) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-26b901c-8f92142 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./dm/tests/download-integration-test-binaries.sh release-7.5 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 536 0 --:--:-- --:--:-- --:--:-- 539 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1021 0 --:--:-- --:--:-- --:--:-- 1025 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1062 0 --:--:-- --:--:-- --:--:-- 1078 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 925 0 --:--:-- --:--:-- --:--:-- 931 Download binaries... >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef/centos7/tidb-server.tar.gz 2024-05-30 15:36:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef/centos7/tidb-server.tar.gz [512424861/512424861] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/634e05a87ee0459846e48e8acea3f6bc5f6973a7/centos7/pd-server.tar.gz 2024-05-30 15:36:44 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/634e05a87ee0459846e48e8acea3f6bc5f6973a7/centos7/pd-server.tar.gz [173743994/173743994] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/1a2c88dcc22c62e1f138d440956214ce367e838b/centos7/tikv-server.tar.gz 2024-05-30 15:37:33 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/1a2c88dcc22c62e1f138d440956214ce367e838b/centos7/tikv-server.tar.gz [817702020/817702020] -> "tmp/tikv-server.tar.gz" [1] >>> download tidb-tools.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/f91f83777df82a9aeb9c3f44898b286851345ade/centos7/tidb-tools.tar.gz 2024-05-30 15:37:48 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/f91f83777df82a9aeb9c3f44898b286851345ade/centos7/tidb-tools.tar.gz [191237316/191237316] -> "tmp/tidb-tools.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-30 15:37:55 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download gh-ost-binary-linux-20200828140552.tar.gz from https://github.com/github/gh-ost/releases/download/v1.1.0/gh-ost-binary-linux-20200828140552.tar.gz 2024-05-30 15:38:01 URL:https://objects.githubusercontent.com/github-production-release-asset-2e65be/54378638/ac8d1400-e94a-11ea-9664-8b516d434420?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=releaseassetproduction%2F20240530%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240530T073553Z&X-Amz-Expires=300&X-Amz-Signature=8e4b0b33e212aef0391ac597ea5954c2dfaafd218aa91dd8b0e4328036b93ae9&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=54378638&response-content-disposition=attachment%3B%20filename%3Dgh-ost-binary-linux-20200828140552.tar.gz&response-content-type=application%2Foctet-stream [5141587/5141587] -> "tmp/gh-ost-binary-linux-20200828140552.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.1G drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:38 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:38 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 15:37 bin -rwxr-xr-x. 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M May 28 20:27 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 43M May 28 20:27 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M May 28 20:27 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 pd-recover -rwxr-xr-x. 1 jenkins jenkins 103M May 28 20:26 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M May 28 20:27 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 regions-dump -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 stores-dump -rwxr-xr-x. 1 jenkins jenkins 131M May 29 09:56 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 29 23:11 tidb-server -rwxr-xr-x. 1 jenkins jenkins 357M May 28 18:52 tikv-server + cd - /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/gh-ost ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tikv-server ./bin/ + ls -alh ./bin total 1.1G drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:38 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:38 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 15:37 bin -rwxr-xr-x. 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 33M May 28 20:27 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 43M May 28 20:27 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M May 28 20:27 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 pd-recover -rwxr-xr-x. 1 jenkins jenkins 103M May 28 20:26 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M May 28 20:27 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 regions-dump -rwxr-xr-x. 1 jenkins jenkins 29M May 28 20:27 stores-dump -rwxr-xr-x. 1 jenkins jenkins 131M May 29 09:56 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 29 23:11 tidb-server -rwxr-xr-x. 1 jenkins jenkins 357M May 28 18:52 tikv-server + ./bin/tidb-server -V Release Version: v7.5.1-164-g39ea2b30d3 Edition: Community Git Commit Hash: 39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef Git Branch: release-7.5 UTC Build Time: 2024-05-29 15:11:39 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v7.5.1-15-g634e05a87 Edition: Community Git Commit Hash: 634e05a87ee0459846e48e8acea3f6bc5f6973a7 Git Branch: release-7.5 UTC Build Time: 2024-05-28 12:26:04 + ./bin/tikv-server -V TiKV Release Version: 7.5.2 Edition: Community Git Commit Hash: 1a2c88dcc22c62e1f138d440956214ce367e838b Git Commit Branch: release-7.5 UTC Build Time: 2024-05-28 10:03:11 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure Profile: dist_release [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (binary/pingcap/tiflow/dm-integration-test/rev-26b901c-8f92142) 1726403072 bytes in 3.14 secs (549298556 bytes/sec) [Pipeline] { [Pipeline] sh + [[ ! -f bin/dm-master.test ]] + [[ ! -f bin/dm-test-tools/check_master_online ]] + [[ ! -f bin/dm-test-tools/check_worker_online ]] + echo 'Binaries already exist, skipping build...' Binaries already exist, skipping build... + ls -alh ./bin total 1.1G drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:38 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:38 .. -rwxr-xr-x. 1 jenkins jenkins 290M May 30 14:47 dm-master.test -rwxr-xr-x. 1 jenkins jenkins 275M May 30 14:50 dm-syncer.test drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 15:38 dm-test-tools -rwxr-xr-x. 1 jenkins jenkins 286M May 30 14:46 dm-worker.test -rwxr-xr-x. 1 jenkins jenkins 209M May 30 14:49 dmctl.test + ls -alh ./bin/dm-test-tools total 588M drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 15:38 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:38 .. -rwxr-xr-x. 1 jenkins jenkins 197M May 30 14:51 check_exit_safe_binlog -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_master_http_apis -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_master_online -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_master_online_http -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_worker_online + which ./bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test + which ./bin/dm-syncer.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test + which ./bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test + which ./bin/dmctl.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dmctl.test + which ./bin/dm-test-tools/check_master_online /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_master_online + which ./bin/dm-test-tools/check_worker_online /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_worker_online [Pipeline] } Cache not saved (binary/pingcap/tiflow/dm-integration-test/rev-26b901c-8f92142 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/gh-ost ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tikv-server ./bin/ + ls -alh ./bin total 2.1G drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:38 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:38 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 15:38 bin -rwxr-xr-x. 1 jenkins jenkins 290M May 30 14:47 dm-master.test -rwxr-xr-x. 1 jenkins jenkins 275M May 30 14:50 dm-syncer.test drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 15:38 dm-test-tools -rwxr-xr-x. 1 jenkins jenkins 286M May 30 14:46 dm-worker.test -rwxr-xr-x. 1 jenkins jenkins 209M May 30 14:49 dmctl.test -rwxr-xr-x. 1 jenkins jenkins 9.8M May 30 15:38 gh-ost -rwxr-xr-x. 1 jenkins jenkins 50M May 30 15:38 minio -rwxr-xr-x. 1 jenkins jenkins 33M May 30 15:38 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 43M May 30 15:38 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 30M May 30 15:38 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 30 15:38 pd-recover -rwxr-xr-x. 1 jenkins jenkins 103M May 30 15:38 pd-server -rwxr-xr-x. 1 jenkins jenkins 25M May 30 15:38 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 29M May 30 15:38 regions-dump -rwxr-xr-x. 1 jenkins jenkins 29M May 30 15:38 stores-dump -rwxr-xr-x. 1 jenkins jenkins 131M May 30 15:38 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 199M May 30 15:38 tidb-server -rwxr-xr-x. 1 jenkins jenkins 357M May 30 15:38 tikv-server + ls -alh ./bin/dm-test-tools total 588M drwxr-sr-x. 2 jenkins jenkins 4.0K May 30 15:38 . drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:38 .. -rwxr-xr-x. 1 jenkins jenkins 197M May 30 14:51 check_exit_safe_binlog -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_master_http_apis -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_master_online -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_master_online_http -rwxr-xr-x. 1 jenkins jenkins 98M May 30 14:51 check_worker_online [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 58.84 secs (51836438 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-813776b1-6117-40cb-a081-9c9645bd187f-wtdl8-whc14 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-faba39cb-b414-4488-b864-3979714390ec-kmlcf-dmx35 Created Pod: kubernetes jenkins-tiflow/dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj-sd2n5 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb [Pipeline] node Agent dm-it-813776b1-6117-40cb-a081-9c9645bd187f-wtdl8-whc14 is provisioned from template dm-it-813776b1-6117-40cb-a081-9c9645bd187f-wtdl8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7527c7bba89bb2b93f1d3f49573ca6b9bfcaed71" jenkins/label: "dm-it-813776b1-6117-40cb-a081-9c9645bd187f" name: "dm-it-813776b1-6117-40cb-a081-9c9645bd187f-wtdl8-whc14" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-813776b1-6117-40cb-a081-9c9645bd187f-wtdl8-whc14" - name: "JENKINS_NAME" value: "dm-it-813776b1-6117-40cb-a081-9c9645bd187f-wtdl8-whc14" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-813776b1-6117-40cb-a081-9c9645bd187f-wtdl8-whc14 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { Agent dm-it-faba39cb-b414-4488-b864-3979714390ec-kmlcf-dmx35 is provisioned from template dm-it-faba39cb-b414-4488-b864-3979714390ec-kmlcf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "af35dd62a9572bb3559a460a13ebdc106182cecc" jenkins/label: "dm-it-faba39cb-b414-4488-b864-3979714390ec" name: "dm-it-faba39cb-b414-4488-b864-3979714390ec-kmlcf-dmx35" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-faba39cb-b414-4488-b864-3979714390ec-kmlcf-dmx35" - name: "JENKINS_NAME" value: "dm-it-faba39cb-b414-4488-b864-3979714390ec-kmlcf-dmx35" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] checkout Running on dm-it-faba39cb-b414-4488-b864-3979714390ec-kmlcf-dmx35 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git The recommended git tool is: git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-146edf55-29c5-4f0f-9fa0-950404364d88-stxrc-5tx0m [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf-051tf [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e94da0b; decorates RemoteLauncher[hudson.remoting.Channel@6c87b69b:JNLP4-connect connection from 10.233.126.90/10.233.126.90:59750] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5869164b; decorates RemoteLauncher[hudson.remoting.Channel@9c28b46:JNLP4-connect connection from 10.233.70.151/10.233.70.151:37188] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] node Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. Created Pod: kubernetes jenkins-tiflow/dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r-sl8rm [Pipeline] podTemplate [Pipeline] { [Pipeline] node > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] withEnv [Pipeline] { Agent dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj-sd2n5 is provisioned from template dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2fcf01026adeb72c5fc486d9c445b737b9c91a53" jenkins/label: "dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164" name: "dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj-sd2n5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj-sd2n5" - name: "JENKINS_NAME" value: "dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj-sd2n5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] container [Pipeline] { Running on dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj-sd2n5 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] { [Pipeline] checkout Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" The recommended git tool is: git [Pipeline] withEnv [Pipeline] { [Pipeline] stage [Pipeline] { (Test) Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] container [Pipeline] { [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { Agent dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb is provisioned from template dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cbc9558acf120c33a8481c5bdfeb433aafd9cb21" jenkins/label: "dm-it-9682e5ac-4e5a-4874-b159-30f340adb447" name: "dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb" - name: "JENKINS_NAME" value: "dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] withCredentials [Pipeline] { Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] checkout > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] { The recommended git tool is: git [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] container [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2dad550a; decorates RemoteLauncher[hudson.remoting.Channel@2c6dfe01:JNLP4-connect connection from 10.233.70.143/10.233.70.143:36038] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 30 07:39 . drwxr-sr-x 17 1000 1000 4.0K May 30 07:39 .. -rw------- 1 root 1000 1.7K May 30 07:39 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:39 ca.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:39 client-cert.pem -rw------- 1 root 1000 1.7K May 30 07:39 client-key.pem -rw------- 1 root 1000 1.7K May 30 07:39 private_key.pem -rw-r--r-- 1 root 1000 451 May 30 07:39 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:39 server-cert.pem -rw------- 1 root 1000 1.7K May 30 07:39 server-key.pem Cloning repository https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0 [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] node + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 30 07:39 . drwxr-sr-x. 17 1000 1000 4.0K May 30 07:39 .. -rw-------. 1 root 1000 1.7K May 30 07:39 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 ca.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 client-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 client-key.pem -rw-------. 1 root 1000 1.7K May 30 07:39 private_key.pem -rw-r--r--. 1 root 1000 451 May 30 07:39 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 server-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 server-key.pem [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@a35885a; decorates RemoteLauncher[hudson.remoting.Channel@53501f24:JNLP4-connect connection from 10.233.68.132/10.233.68.132:35852] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent dm-it-146edf55-29c5-4f0f-9fa0-950404364d88-stxrc-5tx0m is provisioned from template dm-it-146edf55-29c5-4f0f-9fa0-950404364d88-stxrc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "12291db028f0f09becb18f6d1e43508fb9b1d1b1" jenkins/label: "dm-it-146edf55-29c5-4f0f-9fa0-950404364d88" name: "dm-it-146edf55-29c5-4f0f-9fa0-950404364d88-stxrc-5tx0m" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-146edf55-29c5-4f0f-9fa0-950404364d88-stxrc-5tx0m" - name: "JENKINS_NAME" value: "dm-it-146edf55-29c5-4f0f-9fa0-950404364d88-stxrc-5tx0m" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-146edf55-29c5-4f0f-9fa0-950404364d88-stxrc-5tx0m in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Created Pod: kubernetes jenkins-tiflow/dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm-mlq8d Obtained pipelines/pingcap/tiflow/release-7.5/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3 is provisioned from template dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "416e106974726b48c9139ac32deb9811d6918245" jenkins/label: "dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f" name: "dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3" - name: "JENKINS_NAME" value: "dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 6.43 secs (474359413 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] } [Pipeline] sh [Pipeline] node [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] // container Still waiting to schedule task โ€˜dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf-051tfโ€™ is offline [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:39 . drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:39 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 30 15:39 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:39 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:39 .github -rw-r--r-- 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r-- 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r-- 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r-- 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:39 bin drwxr-sr-x 18 jenkins jenkins 4.0K May 30 15:39 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:39 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 30 15:39 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:39 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 30 15:39 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:39 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 30 15:39 engine -rwxr-xr-x 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:39 examples -rw-r--r-- 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r-- 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:39 metrics drwxr-sr-x 49 jenkins jenkins 4.0K May 30 15:39 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:39 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:39 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:39 tests -rw-r--r-- 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:39 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] checkout [Pipeline] cache Agent dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs is provisioned from template dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0f1586cb5caefed319e521e16875ce7bbf746094" jenkins/label: "dm-it-33c68001-94a4-4ffc-b622-0669e37e0319" name: "dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs" - name: "JENKINS_NAME" value: "dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@44878ebd; decorates RemoteLauncher[hudson.remoting.Channel@744f6e0:JNLP4-connect connection from 10.233.69.16/10.233.69.16:35394] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g Still waiting to schedule task โ€˜dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r-sl8rmโ€™ is offline > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Still waiting to schedule task โ€˜dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbhโ€™ is offline Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Still waiting to schedule task โ€˜dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0โ€™ is offline > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Still waiting to schedule task โ€˜dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm-mlq8dโ€™ is offline Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 6.01 secs (507302888 bytes/sec) [Pipeline] { [Pipeline] checkout [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { The recommended git tool is: git [Pipeline] sh Agent dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r-sl8rm is provisioned from template dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6b3316b4bcff1b170e6467934cc04dcdd0523a10" jenkins/label: "dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2" name: "dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r-sl8rm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r-sl8rm" - name: "JENKINS_NAME" value: "dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r-sl8rm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-d75cf82d-8fd4-4de3-a2b3-45554f6a86d2-n276r-sl8rm in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] // timeout + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:39 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:39 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 30 15:39 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:39 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:39 .github -rw-r--r--. 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r--. 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r--. 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r--. 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:39 bin drwxr-sr-x. 18 jenkins jenkins 4.0K May 30 15:39 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:39 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 30 15:39 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:39 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 30 15:39 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:39 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 30 15:39 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:39 examples -rw-r--r--. 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r--. 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:39 metrics drwxr-sr-x. 49 jenkins jenkins 4.0K May 30 15:39 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:39 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:39 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:39 tests -rw-r--r--. 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:39 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh [Pipeline] { [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@b83ead1; decorates RemoteLauncher[hudson.remoting.Channel@6fb61c40:JNLP4-connect connection from 10.233.90.148/10.233.90.148:43382] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials The recommended git tool is: git Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials + '[' TLS_GROUP == G01 ']' + echo 'run G01 test' run G01 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] { [Pipeline] // timeout [Pipeline] sh + make dm_integration_test_in_group GROUP=G01 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh + '[' TLS_GROUP == G00 ']' + echo 'run G00 test' run G00 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh + make dm_integration_test_in_group GROUP=G00 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G01 Run cases: ha_cases3 ha_cases3_1 ha_master ... No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7c789af2; decorates RemoteLauncher[hudson.remoting.Channel@14e7ae29:JNLP4-connect connection from 10.233.68.191/10.233.68.191:33672] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 30 07:39 . drwxr-sr-x. 17 1000 1000 4.0K May 30 07:39 .. -rw-------. 1 root 1000 1.7K May 30 07:39 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 ca.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 client-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 client-key.pem -rw-------. 1 root 1000 1.7K May 30 07:39 private_key.pem -rw-r--r--. 1 root 1000 451 May 30 07:39 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 server-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] { rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@14f96bcf; decorates RemoteLauncher[hudson.remoting.Channel@3177b472:JNLP4-connect connection from 10.233.105.116/10.233.105.116:58764] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] container [Pipeline] { [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 30 07:39 . drwxr-sr-x. 17 1000 1000 4.0K May 30 07:39 .. -rw-------. 1 root 1000 1.7K May 30 07:39 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 ca.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 client-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 client-key.pem -rw-------. 1 root 1000 1.7K May 30 07:39 private_key.pem -rw-r--r--. 1 root 1000 451 May 30 07:39 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 server-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 server-key.pem Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G00 [Pipeline] } [Pipeline] } + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 30 07:39 . drwxr-sr-x. 17 1000 1000 4.0K May 30 07:39 .. -rw-------. 1 root 1000 1.7K May 30 07:39 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 ca.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 client-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 client-key.pem -rw-------. 1 root 1000 1.7K May 30 07:39 private_key.pem -rw-r--r--. 1 root 1000 451 May 30 07:39 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:39 server-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:39 server-key.pem [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { Run cases: ha_cases_1 ha_cases_2 ha_cases2 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh... Verbose mode = false > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:39:43 CST 2024] <<<<<< start test_stop_task >>>>>> [Thu May 30 15:39:43 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:39:44 CST 2024] <<<<<< start test_kill_master >>>>>> [Thu May 30 15:39:44 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Still waiting to schedule task โ€˜dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5gโ€™ is offline > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 import prepare data start DM worker and master [Thu May 30 15:39:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master1.toml >>>>>> [Thu May 30 15:39:46 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master2.toml >>>>>> [Thu May 30 15:39:46 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time import prepare data start DM worker and master cluster [Thu May 30 15:39:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Thu May 30 15:39:46 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Thu May 30 15:39:46 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Thu May 30 15:39:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source1.yaml" [Thu May 30 15:39:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source2.yaml" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Thu May 30 15:39:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:39:52 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" rpc addr 127.0.0.1:8264 is alive [Thu May 30 15:39:53 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time [Thu May 30 15:39:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:18262 is alive [Thu May 30 15:39:54 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml " start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data check diff successfully [Thu May 30 15:40:00 CST 2024] <<<<<< finish test_running >>>>>> kill dm-master1 wait process dm-master1 exit... use sync_diff_inspector to check increment data wait process dm-master1 exit... process dm-master1 already exit waiting 5 seconds check diff successfully check diff successfully [Thu May 30 15:40:03 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=2 expected=2 start dumping SQLs into source stop tasks test dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "query-status test" got=1 expected=1 stop tasks test2 dmctl test cmd: "stop-task test2" got=3 expected=3 dmctl test cmd: "query-status test2" got=1 expected=1 dmctl test cmd: "list-member --worker" got=2 expected=2 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml" check task is running HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive check master2,3 are running dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml" dmctl test cmd: "query-status test2" use sync_diff_inspector to check increment2 data now! check diff successfully [Thu May 30 15:40:10 CST 2024] <<<<<< finish test_kill_master >>>>>> [Thu May 30 15:40:10 CST 2024] <<<<<< start test_kill_and_isolate_worker >>>>>> [Thu May 30 15:40:10 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=4 expected=4 wait process dm-master.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 26.05 secs (117068385 bytes/sec) [Pipeline] { [Pipeline] cache wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... Agent dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g is provisioned from template dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "03809d7a03b8aacc4b2af3f8035558feb53a9df9" jenkins/label: "dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13" name: "dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g" - name: "JENKINS_NAME" value: "dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 6.76 secs (450900751 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] sh import prepare data start DM worker and master cluster [Thu May 30 15:40:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Thu May 30 15:40:21 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Thu May 30 15:40:21 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 30 15:39 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:39 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:39 .github -rw-r--r--. 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r--. 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r--. 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r--. 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 bin drwxr-sr-x. 18 jenkins jenkins 4.0K May 30 15:40 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 30 15:40 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 30 15:40 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:40 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 30 15:40 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 examples -rw-r--r--. 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r--. 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 metrics drwxr-sr-x. 49 jenkins jenkins 4.0K May 30 15:40 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 tests -rw-r--r--. 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh [Pipeline] cache + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 30 15:40 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:40 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 .github -rw-r--r--. 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r--. 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r--. 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r--. 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 bin drwxr-sr-x. 18 jenkins jenkins 4.0K May 30 15:40 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 30 15:40 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 30 15:40 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:40 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 30 15:40 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 examples -rw-r--r--. 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r--. 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 metrics drwxr-sr-x. 49 jenkins jenkins 4.0K May 30 15:40 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 tests -rw-r--r--. 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Thu May 30 15:40:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" [Thu May 30 15:40:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 7.35 secs (414838913 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 30 15:40 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:40 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 .github -rw-r--r--. 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r--. 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r--. 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r--. 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 bin drwxr-sr-x. 18 jenkins jenkins 4.0K May 30 15:40 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 30 15:40 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 30 15:40 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:40 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 30 15:40 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 examples -rw-r--r--. 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r--. 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 metrics drwxr-sr-x. 49 jenkins jenkins 4.0K May 30 15:40 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 tests -rw-r--r--. 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } [Pipeline] { [Pipeline] checkout [Pipeline] stage [Pipeline] { (Test) The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] // timeout [Pipeline] sh use sync_diff_inspector to check increment data check diff successfully [Thu May 30 15:40:35 CST 2024] <<<<<< finish test_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-02 worker2" [Pipeline] // timeout [Pipeline] sh + '[' TLS_GROUP == G03 ']' + echo 'run G03 test' run G03 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] } [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { + make dm_integration_test_in_group GROUP=G03 + '[' TLS_GROUP == G04 ']' + echo 'run G04 test' run G04 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] // timeout [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@746b8e7a; decorates RemoteLauncher[hudson.remoting.Channel@32560fd9:JNLP4-connect connection from 10.233.67.25/10.233.67.25:51022] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { + make dm_integration_test_in_group GROUP=G04 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + '[' TLS_GROUP == G05 ']' + echo 'run G05 test' run G05 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh got=2 expected=2 kill dm-worker2 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G03 Run cases: dmctl_advance dmctl_basic dmctl_command ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + make dm_integration_test_in_group GROUP=G05 [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G04 Run cases: import_goroutine_leak incremental_mode initial_unit import_goroutine_leak not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 30 07:40 . drwxr-sr-x 17 1000 1000 4.0K May 30 07:40 .. -rw------- 1 root 1000 1.7K May 30 07:40 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:40 ca.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:40 client-cert.pem -rw------- 1 root 1000 1.7K May 30 07:40 client-key.pem -rw------- 1 root 1000 1.7K May 30 07:40 private_key.pem -rw-r--r-- 1 root 1000 451 May 30 07:40 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:40 server-cert.pem -rw------- 1 root 1000 1.7K May 30 07:40 server-key.pem Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] sh wait process dm-worker2 exit... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G05 Run cases: load_interrupt many_tables online_ddl load_interrupt not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 30 07:40 . drwxr-sr-x. 17 1000 1000 4.0K May 30 07:40 .. -rw-------. 1 root 1000 1.7K May 30 07:40 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:40 ca.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:40 client-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:40 client-key.pem -rw-------. 1 root 1000 1.7K May 30 07:40 private_key.pem -rw-r--r--. 1 root 1000 451 May 30 07:40 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:40 server-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:40 server-key.pem [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status test" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. got=1 expected=1 [Thu May 30 15:40:38 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" version() 5.7.26-log version() 8.0.21 start running case: [dmctl_advance] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm [Thu May 30 15:40:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [incremental_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:40:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8264 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Thu May 30 15:40:40 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [many_tables] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 30 07:40 . drwxr-sr-x 17 1000 1000 4.0K May 30 07:40 .. -rw------- 1 root 1000 1.7K May 30 07:40 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:40 ca.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:40 client-cert.pem -rw------- 1 root 1000 1.7K May 30 07:40 client-key.pem -rw------- 1 root 1000 1.7K May 30 07:40 private_key.pem -rw-r--r-- 1 root 1000 451 May 30 07:40 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:40 server-cert.pem -rw------- 1 root 1000 1.7K May 30 07:40 server-key.pem wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3 worker4" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock unlock" dmctl test cmd: "shard-ddl-lock unlock lock_id --force-remove=invalid-force-remove" dmctl test cmd: "handle-error" dmctl test cmd: "handle-error test-task --binlog-pos mysql-bin:shoud-bin-digital skip" dmctl test cmd: "handle-error test-task replace alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "handle-error test-task wrong_operation" dmctl test cmd: "binlog" dmctl test cmd: "binlog skip test-task --binlog-pos mysql-bin:should-be-digital" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:40:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "binlog replace test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog inject test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog list test-task --binlog-pos mysql-bin:should-be-digital" dmctl test cmd: "binlog wrong_operation test-task" got=3 expected=3 restart dm-worker3 dmctl test cmd: "binlog-schema" dmctl test cmd: "binlog-schema test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:40:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog-schema list test" dmctl test cmd: "binlog-schema update test" dmctl test cmd: "binlog-schema update test test_db test_table --from-source" dmctl test cmd: "binlog-schema update test test_db test_table --from-source --from-target" wait process dm-worker3 exit... dmctl test cmd: "binlog-schema update test test_db test_table schema.sql --from-source" dmctl test cmd: "binlog-schema delete test" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive wait process dm-worker3 exit... process dm-worker3 already exit wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Thu May 30 15:40:44 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-master.test exit... start prepare_data rpc addr 127.0.0.1:8264 is alive isolate dm-worker4 kill dm-worker4 wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:40:45 CST 2024] <<<<<< test case dmctl_advance success! >>>>>> start running case: [dmctl_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm dmctl test cmd: "any command" wait process dm-worker4 exit... dmctl test cmd: "list-member" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 6.97 secs (437419832 bytes/sec) [Pipeline] { [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Thu May 30 15:40:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker4 exit... process dm-worker4 already exit [Thu May 30 15:40:47 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 isolate dm-worker3 kill dm-worker3 [Pipeline] } [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] cache rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:40:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker3 exit... dmctl test cmd: "any command" [Thu May 30 15:40:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" wait process dm-worker3 exit... process dm-worker3 already exit dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" [Thu May 30 15:40:50 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 disable isolate dm-worker4 kill dm-worker4 rpc addr 127.0.0.1:8261 is alive check_task_wrong_arg dmctl test cmd: "check-task" dmctl test cmd: "check-task not_exists_config_file" pause_relay_wrong_arg dmctl test cmd: "pause-relay wrong_arg" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "pause-relay" resume_relay_wrong_arg dmctl test cmd: "resume-relay wrong_arg" dmctl test cmd: "resume-relay" pause_task_wrong_arg dmctl test cmd: "pause-task" dmctl test cmd: "pause-task -s a -s b" resume_task_wrong_arg dmctl test cmd: "resume-task" dmctl test cmd: "resume-task -s a -s b" query_status_wrong_arg dmctl test cmd: "query-status wrong_args_count more_than_one" wait process dm-worker4 exit... dmctl test cmd: "query-status -s source-x task-y" start_task_wrong_arg dmctl test cmd: "start-task" dmctl test cmd: "start-task not_exists_config_file" stop_task_wrong_arg dmctl test cmd: "stop-task" dmctl test cmd: "stop-task -s a -s b" show_ddl_locks_wrong_arg dmctl test cmd: "shard-ddl-lock a b" dmctl test cmd: "show-ddl-locks a b" wait process dm-worker4 exit... process dm-worker4 already exit [Thu May 30 15:40:52 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" purge_relay_wrong_arg dmctl test cmd: "purge-relay wrong_arg" dmctl test cmd: "purge-relay" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=1 expected=3 command: query-status test "stage": "Running" count: 1 != expected: 3, failed the 0-th time, will retry again dmctl test cmd: "purge-relay --filename bin-000001 -s mysql-replica-01 -s mysql-replica-02" operate_source_empty_arg dmctl test cmd: "operate-source" dmctl test cmd: "operate-source create not_exists_config_file" dmctl test cmd: "operate-source invalid /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" dmctl test cmd: "operate-source stop /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" transfer_source_empty_arg dmctl test cmd: "transfer-source" dmctl test cmd: "transfer-source source-id" dmctl test cmd: "transfer-source source-id worker-id another-worker" got=2 expected=2 start_relay_empty_arg dmctl test cmd: "start-relay" dmctl test cmd: "start-relay wrong_arg" stop_relay_empty_arg dmctl test cmd: "stop-relay" dmctl test cmd: "stop-relay wrong_arg" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" source-id: mysql-replica-01 server-id: 123456 enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' enable-relay: false from: host: 127.0.0.1 user: root port: 3306 filters: - schema-pattern: "dmctl" table-pattern: "t_1" sql-pattern: ["alter table .* add column aaa int"] action: Ignore - schema-pattern: "dmctl" table-pattern: "t_2" sql-pattern: ["alter table .* add column aaa int"] action: Ignore dmctl test cmd: "operate-source stop /tmp/dm_test/dmctl_basic/source1-wrong.yaml mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" [Thu May 30 15:40:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=3 expected=3 query-status from all dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "pause-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" restart worker4 dmctl test cmd: "operate-source show" [Thu May 30 15:40:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker4 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" dmctl test cmd: "operate-source show -s mysql-replica-01" ############################################################ test case for operate tasks bound to a source ############################################################ dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/foo-task.yaml" wait process dm-worker4 exit... process dm-worker4 already exit [Thu May 30 15:40:57 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> Agent dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf-051tf is provisioned from template dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c812cc43699dd42fa0e63777e8d65cd3a5e4dad7" jenkins/label: "dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4" name: "dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf-051tf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf-051tf" - name: "JENKINS_NAME" value: "dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf-051tf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" wait for rpc addr 127.0.0.1:18262 alive the 1-th time Running on dm-it-7094e88f-9194-4f2d-a36e-42eff8846fc4-nvcwf-051tf in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test dmctl test cmd: "list-member" [Thu May 30 15:40:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "resume-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 7.08 secs (430495931 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] sh [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:40:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:40 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 30 15:40 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:40 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 .github -rw-r--r--. 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r--. 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r--. 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r--. 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 bin drwxr-sr-x. 18 jenkins jenkins 4.0K May 30 15:40 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 30 15:40 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 30 15:40 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:40 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 30 15:40 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 examples -rw-r--r--. 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r--. 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:40 metrics drwxr-sr-x. 49 jenkins jenkins 4.0K May 30 15:40 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:40 tests -rw-r--r--. 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "pause-task -s xxx" dmctl test cmd: "resume-task -s xxx" dmctl test cmd: "stop-task -s xxx" finish prepare_data [Thu May 30 15:40:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:41:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> dmctl test cmd: "pause-task -s mysql-replica-01" wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive [Pipeline] container [Pipeline] { [Pipeline] cache use sync_diff_inspector to check increment2 data now! check diff successfully [Thu May 30 15:41:01 CST 2024] <<<<<< finish test_kill_and_isolate_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status -s mysql-replica-01" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/many_tables/source1.yaml" dmctl test cmd: "query-status -s mysql-replica-02" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:40 . drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:40 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 30 15:40 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:40 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:40 .github -rw-r--r-- 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r-- 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r-- 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r-- 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:40 bin drwxr-sr-x 18 jenkins jenkins 4.0K May 30 15:40 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:40 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 30 15:40 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:40 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 30 15:40 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:40 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 30 15:40 engine -rwxr-xr-x 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:40 examples -rw-r--r-- 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r-- 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:40 metrics drwxr-sr-x 49 jenkins jenkins 4.0K May 30 15:40 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:40 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:40 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:40 tests -rw-r--r-- 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task.yaml " dmctl test cmd: "pause-task -s mysql-replica-02" wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" wait process dm-master.test exit... dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source1.yaml" wait process dm-master.test exit... dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-task -s mysql-replica-01" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=2 expected=2 start task in incremental mode with zero gtid/pos [Thu May 30 15:41:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-master.test exit... dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:41:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" 0 in Load unit "progress": "0.00 %", dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:10 CST 2024] <<<<<< test case ha_cases_1 success! >>>>>> start running case: [ha_cases_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:10 CST 2024] <<<<<< start test_kill_master_in_sync >>>>>> [Thu May 30 15:41:10 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/baz-task.yaml" wait process dm-worker.test exit... dmctl test cmd: "pause-task -s mysql-replica-01" wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:41:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time import prepare data start DM worker and master cluster [Thu May 30 15:41:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Thu May 30 15:41:12 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Thu May 30 15:41:12 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:41:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "resume-task -s mysql-replica-01" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Thu May 30 15:41:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time 0 in Load unit "progress": "0.00 %", rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" [Thu May 30 15:41:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "transfer-source mysql-replica-01 worker1" dmctl test cmd: "transfer-source mysql-replica-01 worker2" kill worker2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" [2024/05/30 15:41:09.004 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b847231f-1e57-11ef-b8bd-5e469546a4cf.000001/dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb-bin.000001] [2024/05/30 15:41:09.031 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b847231f-1e57-11ef-b8bd-5e469546a4cf.000001/dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb-bin.000001] [2024/05/30 15:41:13.759 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b847231f-1e57-11ef-b8bd-5e469546a4cf.000001/dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb-bin.000001] [2024/05/30 15:41:13.778 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b847231f-1e57-11ef-b8bd-5e469546a4cf.000001/dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb-bin.000001] [2024/05/30 15:41:13.788 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b847231f-1e57-11ef-b8bd-5e469546a4cf.000001/dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb-bin.000001] [2024/05/30 15:41:13.810 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=b847231f-1e57-11ef-b8bd-5e469546a4cf.000001/dm-it-9682e5ac-4e5a-4874-b159-30f340adb447-5zj3z-qvtpb-bin.000001] dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "pause-task test" wait process worker2 exit... start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " 0 in Load unit "progress": "22.83 %", wait process worker2 exit... process worker2 already exit dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" restart worker2 [Thu May 30 15:41:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-01 worker2" dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "stop-relay -s mysql-replica-01" pause_relay_success dmctl test cmd: "pause-relay -s mysql-replica-01 -s mysql-replica-02" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 19.24 secs (158499392 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=2 expected=2 dmctl test cmd: "resume-relay -s mysql-replica-01 -s mysql-replica-02" [Pipeline] sh dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl_check_task dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:41 . drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:41 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 30 15:41 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:41 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:41 .github -rw-r--r-- 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r-- 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r-- 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r-- 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:41 bin drwxr-sr-x 18 jenkins jenkins 4.0K May 30 15:41 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:41 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 30 15:41 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:41 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 30 15:41 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:41 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 30 15:41 engine -rwxr-xr-x 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:41 examples -rw-r--r-- 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r-- 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:41 metrics drwxr-sr-x 49 jenkins jenkins 4.0K May 30 15:41 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:41 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:41 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:41 tests -rw-r--r-- 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:41 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] { [Pipeline] } dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml" use sync_diff_inspector to check increment data check diff successfully [Thu May 30 15:41:23 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 100 -w 1" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 0" 0 in Load unit "progress": "51.09 %", dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" wait process dm-master1 exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" check_task_optimistic dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task4.yaml" check_task_only_warning dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" [Pipeline] } [Pipeline] checkout [Pipeline] stage [Pipeline] { (Test) dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" [Pipeline] // timeout [Pipeline] sh The recommended git tool is: git wait process dm-master1 exit... process dm-master1 already exit wait and check task running [Pipeline] } + '[' TLS_GROUP == G08 ']' + echo 'run G08 test' run G08 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] // timeout [Pipeline] sh + make dm_integration_test_in_group GROUP=G08 [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] // timeout [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" check task with empty unit config dmctl test cmd: "check-task /tmp/empty-dump.yaml" --- name: empty-unit-task task-mode: all target-database: host: "127.0.0.1" port: 4000 user: "root" password: "" mysql-instances: - source-id: "mysql-replica-01" mydumper-config-name: "global" loader-config-name: "global" syncer-config-name: "global" block-allow-list: "instance" mydumpers: global: threads: 4 loaders: global: syncers: global: worker-count: 32 block-allow-list: instance: do-dbs: ["dmctl"] do-tables: - db-name: "dmctl" [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] { tbl-name: "~^t_[\\d]+"dmctl test cmd: "check-task /tmp/empty-load.yaml" dmctl test cmd: "check-task /tmp/empty-sync.yaml" [Pipeline] container [Pipeline] { /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G08 [Pipeline] sh start task with empty unit config dmctl test cmd: "start-task /tmp/empty-cfg.yaml" Run cases: shardddl3 shardddl3_1 shardddl4 shardddl4_1 sharding sequence_sharding ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2b68c734; decorates RemoteLauncher[hudson.remoting.Channel@72a644af:JNLP4-connect connection from 10.233.67.109/10.233.67.109:44250] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 30 07:41 . drwxr-sr-x. 17 1000 1000 4.0K May 30 07:41 .. -rw-------. 1 root 1000 1.7K May 30 07:41 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:41 ca.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:41 client-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:41 client-key.pem -rw-------. 1 root 1000 1.7K May 30 07:41 private_key.pem -rw-r--r--. 1 root 1000 451 May 30 07:41 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:41 server-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:41 server-key.pem [Pipeline] } Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache dmctl test cmd: "config task empty-unit-task" dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/empty-unit-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/dmctl_basic/dm-task-error-database-config.yaml" dmctl_start_task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [shardddl3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh... Verbose mode = false + '[' TLS_GROUP == G07 ']' + echo 'run G07 test' run G07 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time + make dm_integration_test_in_group GROUP=G07 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt 0 in Load unit "progress": "76.75 %", dmctl test cmd: "stop-task test" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G07 Run cases: shardddl1 shardddl1_1 shardddl2 shardddl2_1 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + '[' TLS_GROUP == G02 ']' + echo 'run G02 test' run G02 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G02 dmctl test cmd: "query-status test_incremental_no_source_meta" got=3 expected=3 dmctl test cmd: "stop-task test_incremental_no_source_meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml " /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:41:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source1.yaml" check diff successfully dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:41:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:41:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G02 Run cases: handle_error handle_error_2 handle_error_3 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" wait for rpc addr 127.0.0.1:8262 alive the 1-th time Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [shardddl1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:41:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source2.yaml" wait process dmctl_basic/worker1 exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:41:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [handle_error] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:41:34 CST 2024] <<<<<< start DM-071 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive wait process dmctl_basic/worker1 exit... process dmctl_basic/worker1 already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source1.yaml" [Thu May 30 15:41:35 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> 0 in Sync unit wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:41:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dmctl_basic/worker2 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source2.yaml" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:41:36 CST 2024] <<<<<< finish DM-071 pessimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source1.yaml" wait process dmctl_basic/worker2 exit... process dmctl_basic/worker2 already exit dmctl test cmd: "query-status" got=2 expected=2 got=2 expected=2 [Thu May 30 15:41:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time Agent dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh is provisioned from template dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "78e3cc0982cd706556c00e8a39b8d7f858268472" jenkins/label: "dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9" name: "dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh" - name: "JENKINS_NAME" value: "dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml" [Thu May 30 15:41:37 CST 2024] <<<<<< start DM-073 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" [Thu May 30 15:41:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:41:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 7.33 secs (416270836 bytes/sec) [Pipeline] { wait process dm-worker exit... got=1 expected=1 dmctl test cmd: "resume-task test" Agent dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0 is provisioned from template dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9eaf1e227f3174ccb18fb586226aef570f76c697" jenkins/label: "dm-it-f736575a-2dc6-454c-841b-c486aca56cd8" name: "dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0" - name: "JENKINS_NAME" value: "dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:8263 is alive config dmctl test cmd: "config" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "config hihi haha" dmctl test cmd: "config task haha" dmctl test cmd: "query-status test" wait process dm-worker exit... process dm-worker already exit [Thu May 30 15:41:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Thu May 30 15:41:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "config master haha" dmctl test cmd: "config worker haha" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" [Thu May 30 15:41:39 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR >>>>>> [Thu May 30 15:41:39 CST 2024] <<<<<< finish DM-073 pessimistic >>>>>> dmctl test cmd: "config source haha" dmctl test cmd: "get-config haha" got=1 expected=1 dmctl test cmd: "resume-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" dmctl test cmd: "get-config haha hihi" dmctl test cmd: "get-config master haha" check diff failed 1-th time, retry later [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { check diff successfully dmctl test cmd: "get-config worker haha" dmctl test cmd: "get-config source haha" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:41:40 CST 2024] <<<<<< start DM-COMPACT >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" [Pipeline] { [Pipeline] { dmctl test cmd: "get-config task haha" dmctl test cmd: "get-task-config haha" [Pipeline] sh [Thu May 30 15:41:40 CST 2024] <<<<<< start DM-076 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "get-task-config haha hihi" dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" [Pipeline] withEnv [Pipeline] { [Pipeline] checkout dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" [Pipeline] container [Pipeline] { [Pipeline] checkout The recommended git tool is: git + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:41 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:41 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 30 15:41 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:41 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 .github -rw-r--r--. 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r--. 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r--. 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r--. 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 bin drwxr-sr-x. 18 jenkins jenkins 4.0K May 30 15:41 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 30 15:41 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 30 15:41 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:41 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 30 15:41 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 examples -rw-r--r--. 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r--. 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 metrics drwxr-sr-x. 49 jenkins jenkins 4.0K May 30 15:41 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:41 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:41 tests -rw-r--r--. 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break The recommended git tool is: git [Pipeline] } [Pipeline] // timeout [Pipeline] sh dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" [Pipeline] stage [Pipeline] { (Test) got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb2 add column c int default 100; alter table handle_error.tb2 add primary key (c);" dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "stop-task test" [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { dmctl test cmd: "query-status test" got=1 expected=1 [Pipeline] withCredentials + '[' TLS_GROUP == TLS_GROUP ']' + echo 'run tls test' run tls test + echo 'copy mysql certs' copy mysql certs + sudo mkdir -p /var/lib/mysql + sudo chmod 777 /var/lib/mysql + sudo chown -R 1000:1000 /var/lib/mysql + sudo cp -r /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ca-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ca.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/client-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/client-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/private_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/public_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/server-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/server-key.pem /var/lib/mysql/ + sudo chown -R 1000:1000 /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem + ls -alh /var/lib/mysql/ total 48K drwxrwxrwx. 2 jenkins jenkins 4.0K May 30 15:41 . drwxr-xr-x. 1 root root 4.0K May 30 15:41 .. -rw-------. 1 jenkins jenkins 1.7K May 30 15:41 ca-key.pem -rw-r--r--. 1 jenkins jenkins 1.1K May 30 15:41 ca.pem -rw-r--r--. 1 jenkins jenkins 1.1K May 30 15:41 client-cert.pem -rw-------. 1 jenkins jenkins 1.7K May 30 15:41 client-key.pem -rw-------. 1 jenkins jenkins 1.7K May 30 15:41 private_key.pem -rw-r--r--. 1 jenkins jenkins 451 May 30 15:41 public_key.pem -rw-r--r--. 1 jenkins jenkins 1.1K May 30 15:41 server-cert.pem -rw-------. 1 jenkins jenkins 1.7K May 30 15:41 server-key.pem + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog inject test -s mysql-replica-01 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique(c);" dmctl test cmd: "operate-source stop mysql-replica-01" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source1.yaml" [Pipeline] { [Pipeline] container [Pipeline] { dmctl test cmd: "query-status test" got=1 expected=1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3cc5e313; decorates RemoteLauncher[hudson.remoting.Channel@31dcd281:JNLP4-connect connection from 10.233.97.96/10.233.97.96:41382] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] sh dmctl test cmd: "binlog inject test -s mysql-replica-02 alter table handle_error.tb2 drop primary key; alter table handle_error.tb2 add unique(c);" No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1395b2ce; decorates RemoteLauncher[hudson.remoting.Channel@43a88f13:JNLP4-connect connection from 10.233.90.56/10.233.90.56:47276] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 check diff failed 1-th time, retry later run tidb sql failed 1-th time, retry later check diff successfully + make dm_integration_test_in_group GROUP=TLS_GROUP + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 30 07:41 . drwxr-sr-x. 17 1000 1000 4.0K May 30 07:41 .. -rw-------. 1 root 1000 1.7K May 30 07:41 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:41 ca.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:41 client-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:41 client-key.pem -rw-------. 1 root 1000 1.7K May 30 07:41 private_key.pem -rw-r--r--. 1 root 1000 451 May 30 07:41 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 30 07:41 server-cert.pem -rw-------. 1 root 1000 1.7K May 30 07:41 server-key.pem [Pipeline] } [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt dmctl test cmd: "operate-source stop mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source2.yaml" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh TLS_GROUP Run cases: tls ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully dmctl test cmd: "stop-task test" start incremental_data % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 3650 0 3650 0 0 434k 0 --:--:-- --:--:-- --:--:-- 445k dmctl test cmd: "stop-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" got=2 expected=2 dmctl test cmd: "start-task /tmp/dm_test/dmctl_basic/get_task.yaml" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) [Thu May 30 15:41:44 CST 2024] <<<<<< finish DM-076 pessimistic >>>>>> [Thu May 30 15:41:44 CST 2024] <<<<<< start DM-076 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [tls] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit Avoid second fetch dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" dmctl test cmd: "stop-task test" [Thu May 30 15:41:46 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR >>>>>> [Thu May 30 15:41:46 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master exit... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls check diff failed 1-th time, retry later use sync_diff_inspector to check increment data check diff successfully check diff successfully [Thu May 30 15:41:34 CST 2024] <<<<<< finish test_stop_task >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:43 CST 2024] <<<<<< test case ha_cases3 success! >>>>>> start running case: [ha_cases3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:43 CST 2024] <<<<<< start test_isolate_master_and_worker >>>>>> [Thu May 30 15:41:43 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master [Thu May 30 15:41:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> [Thu May 30 15:41:45 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> [Thu May 30 15:41:45 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Thu May 30 15:41:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master exit... process dm-master already exit [Thu May 30 15:41:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source1.yaml" check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Thu May 30 15:41:49 CST 2024] <<<<<< finish DM-076 optimistic >>>>>> check diff failed 2-th time, retry later [Thu May 30 15:41:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Thu May 30 15:41:50 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:41:50 CST 2024] <<<<<< test case incremental_mode success! >>>>>> start running case: [initial_unit] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh... Verbose mode = false rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source2.yaml" 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check diff failed 3-th time, retry later wait process worker1 exit... [Thu May 30 15:41:52 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 [Thu May 30 15:41:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Thu May 30 15:41:52 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Thu May 30 15:41:52 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process worker1 exit... check diff failed 4-th time, retry later rpc addr 127.0.0.1:8264 is alive [Thu May 30 15:41:53 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Thu May 30 15:41:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker1 exit... rpc addr 127.0.0.1:18262 is alive [Thu May 30 15:41:54 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "pause-task test" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:41:54 CST 2024] <<<<<< finish DM-077 >>>>>> [Thu May 30 15:41:55 CST 2024] <<<<<< start DM-077 >>>>>> wait process worker1 exit... dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task.yaml " check diff failed 5-th time, retry later got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait process worker1 exit... dmctl test cmd: "resume-task test" wait process worker1 exit... dmctl test cmd: "query-status test" got=4 expected=4 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 finish incremental_data check diff 1 Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 10.50 secs (290384386 bytes/sec) [Pipeline] { check master alive dmctl test cmd: "list-member" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" got=4 expected=4 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully dmctl test cmd: "query-status test" check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase got=1 expected=1 wait process worker1 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" check diff failed 6-th time, retry later wait process worker1 exit... [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { failpoint=github.com/pingcap/tiflow/dm/syncer/LoadCheckpointFailed=return(1152) [Thu May 30 15:41:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] sh [Thu May 30 15:41:59 CST 2024] <<<<<< finish DM-077 >>>>>> dmctl test cmd: "query-status test" [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { test http and api interface Release Version: v7.5.1-58-g8f92142d7 Git Commit Hash: 8f92142d7874bfad2a8db6ca100604c383fca9df Git Branch: HEAD UTC Build Time: 2024-05-30 06:42:59 Go Version: go version go1.21.10 linux/amd64 Failpoint Build: true rpc addr 127.0.0.1:8261 is alive + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:41 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 30 15:41 .. -rw-r--r--. 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 30 15:41 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:41 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 .github -rw-r--r--. 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r--. 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r--. 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r--. 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 bin drwxr-sr-x. 18 jenkins jenkins 4.0K May 30 15:41 cdc drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 cdcv2 drwxr-sr-x. 12 jenkins jenkins 4.0K May 30 15:41 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 30 15:41 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 30 15:41 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 30 15:41 engine -rwxr-xr-x. 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 examples -rw-r--r--. 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r--. 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 30 15:41 metrics drwxr-sr-x. 49 jenkins jenkins 4.0K May 30 15:41 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:41 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 30 15:41 tests -rw-r--r--. 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x. 3 jenkins jenkins 4.0K May 30 15:41 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] stage [Pipeline] { (Test) wait process worker1 exit... [Pipeline] stage [Pipeline] { (Test) [Pipeline] } check diff failed 7-th time, retry later [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { { "result": true, "msg": "", "members": [ { "leader": { "msg": "", "name": "master2", "addr": "127.0.0.1:8361" } }, { "master": { "msg": "", "masters": [ { "name": "master1", "memberID": "10068700568285020701", "alive": true, "peerURLs": [ "https://127.0.0.1:8291" ], "clientURLs": [ "https://127.0.0.1:8261" ] }, { "name": "master2", "memberID": "518295880577331676", "alive": true, "peerURLs": [ "https://127.0.0.1:8292" ], "clientURLs": [ "https://127.0.0.1:8361" ] }, { "name": "master3", "memberID": "6800138986734450252", "alive": true, "peerURLs": [ "https://127.0.0.1:8293" ], "clientURLs": [ "https://127.0.0.1:8461" ] } ] } }, { "worker": { "msg": "", "workers": [ { "name": "worker1", "addr": "127.0.0.1:8262", "stage": "bound", "source": "mysql-replica-01" } ] } } ] } rpc addr 127.0.0.1:8261 is alive use common name not in 'cert-allowed-cn' should not request success dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" [Pipeline] // timeout [Pipeline] sh rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:42:00 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process worker1 exit... + '[' TLS_GROUP == G06 ']' + echo 'run G06 test' run G06 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] withCredentials wait for rpc addr 127.0.0.1:8261 alive the 2-th time Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] { [Pipeline] container + make dm_integration_test_in_group GROUP=G06 [Pipeline] { [Pipeline] container [Pipeline] { rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/0/source1.yaml" [Pipeline] sh dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt wait process worker1 exit... process worker1 already exit check diff failed 8-th time, retry later check diff failed 1-th time, retry later [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G06 Run cases: relay_interrupt safe_mode sequence_safe_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" use sync_diff_inspector to check increment data check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 30 07:42 . drwxr-sr-x 17 1000 1000 4.0K May 30 07:42 .. -rw------- 1 root 1000 1.7K May 30 07:42 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 ca.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 client-cert.pem -rw------- 1 root 1000 1.7K May 30 07:42 client-key.pem -rw------- 1 root 1000 1.7K May 30 07:42 private_key.pem -rw-r--r-- 1 root 1000 451 May 30 07:42 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 server-cert.pem -rw------- 1 root 1000 1.7K May 30 07:42 server-key.pem wait for rpc addr 127.0.0.1:8261 alive the 3-th time wait process worker2 exit... check diff successfully [Thu May 30 15:42:02 CST 2024] <<<<<< finish test_multi_task_running >>>>>> [Thu May 30 15:42:02 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 30 07:42 . drwxr-sr-x 17 1000 1000 4.0K May 30 07:42 .. -rw------- 1 root 1000 1.7K May 30 07:42 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 ca.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 client-cert.pem -rw------- 1 root 1000 1.7K May 30 07:42 client-key.pem -rw------- 1 root 1000 1.7K May 30 07:42 private_key.pem -rw-r--r-- 1 root 1000 451 May 30 07:42 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 server-cert.pem -rw------- 1 root 1000 1.7K May 30 07:42 server-key.pem [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { wait for rpc addr 127.0.0.1:8261 alive the 4-th time [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { wait process worker2 exit... process worker2 already exit dmctl test cmd: "list-member --worker" got=2 expected=2 [Thu May 30 15:42:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] cache rpc addr 127.0.0.1:8561 is alive check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:42:04 CST 2024] <<<<<< finish DM-078 >>>>>> [Thu May 30 15:42:04 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" check diff failed 9-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [relay_interrupt] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh... Verbose mode = false wait for rpc addr 127.0.0.1:8261 alive the 5-th time 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetGlobalVariableFailed=return("server_uuid,1152") rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:42:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" resume task will also initial failed dmctl test cmd: "resume-task test" check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 6-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:42:06 CST 2024] <<<<<< finish DM-COMPACT >>>>>> dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 7-th time dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "query-status test" [Thu May 30 15:42:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" show_ddl_locks_no_locks dmctl test cmd: "shard-ddl-lock test" dmctl test cmd: "show-ddl-locks test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" wait process dm-worker exit... check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status" dmctl test cmd: "pause-task test" wait for rpc addr 127.0.0.1:8261 alive the 8-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=2 expected=2 dmctl master_addr dmctl test cmd: "list-member" dmctl test cmd: "list-member" [Thu May 30 15:42:08 CST 2024] <<<<<< finish DM-078 >>>>>> dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success wait process dm-worker exit... process dm-worker already exit [Thu May 30 15:42:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Thu May 30 15:42:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:42:09 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 9-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:42:09 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:42:09 CST 2024] <<<<<< start DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 10-th time check diff successfully [2024/05/30 15:41:02.209 +08:00] [INFO] [dumpling.go:110] ["create dumpling"] [task=test] [unit=dump] [config="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"auto\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"\",\"ServerInfo\":{\"ServerType\":0,\"ServerVersion\":null,\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/30 15:41:02.350 +08:00] [INFO] [dump.go:151] ["begin to run Dump"] [task=test] [unit=dump] [conf="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"flush\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"sql\",\"ServerInfo\":{\"ServerType\":1,\"ServerVersion\":\"5.7.26-log\",\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/30 15:41:12.233 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14211] [2024/05/30 15:41:22.234 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14211] [2024/05/30 15:41:32.234 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14211] [2024/05/30 15:41:42.234 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=9415450] [2024/05/30 15:41:52.235 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=10472763] [2024/05/30 15:42:02.235 +08:00] [DEBUG] [syncer.go:361] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=20038344] check diff 2 rpc addr 127.0.0.1:8661 is alive wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:42:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/0/source1.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" got=1 expected=1 [Thu May 30 15:42:11 CST 2024] <<<<<< finish DM-079 >>>>>> [Thu May 30 15:42:11 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 11-th time rpc addr 127.0.0.1:8262 is alive Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 5.27 secs (578789130 bytes/sec) [Pipeline] { [Pipeline] cache run tidb sql failed 1-th time, retry later dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Thu May 30 15:42:11 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> [Thu May 30 15:42:11 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" waiting for asynchronous relay and subtask to be started dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 12-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" Agent dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm-mlq8d is provisioned from template dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/545/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cef9e27e408420bcb13fd9d8ecdceda4fa4718d5" jenkins/label: "dm-it-3cf20359-a013-4317-a946-7a63929b04c0" name: "dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm-mlq8d" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm-mlq8d" - name: "JENKINS_NAME" value: "dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm-mlq8d" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-3cf20359-a013-4317-a946-7a63929b04c0-pt6fm-mlq8d in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test got=1 expected=1 dmctl test cmd: "stop-task test" check log contain failed 1-th time, retry later [Thu May 30 15:42:13 CST 2024] <<<<<< finish DM-079 >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 13-th time run tidb sql failed 2-th time, retry later dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" [Thu May 30 15:42:14 CST 2024] <<<<<< start DM-080 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status" reset go failpoints, and need restart dm-worker then resume task, task will recover success wait for rpc addr 127.0.0.1:8261 alive the 14-th time dmctl test cmd: "list-member" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" check diff successfully try to isolate leader dm-master2 kill dm-master2 [Thu May 30 15:42:15 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> [Thu May 30 15:42:15 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 15-th time dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... run tidb sql failed 3-th time, retry later wait process dm-master2 exit... wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:42:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 16-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:42:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master2 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01,mysql-replica-02 alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" wait for rpc addr 127.0.0.1:8261 alive the 17-th time dmctl test cmd: "query-status test" got=4 expected=4 rpc addr 127.0.0.1:8262 is alive run tidb sql failed 4-th time, retry later run tidb sql failed 1-th time, retry later wait process dm-master2 exit... dmctl test cmd: "stop-task test" process dm-master2 already exit [Thu May 30 15:42:18 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> try to isolate follower dm-master1 kill dm-master1 wait for rpc addr 127.0.0.1:8261 alive the 18-th time wait process dm-master1 exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 5.80 secs (525409648 bytes/sec) [Pipeline] { rpc addr 127.0.0.1:8262 is alive run tidb sql failed 5-th time, retry later dmctl test cmd: "stop-task test" [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { wait for rpc addr 127.0.0.1:8261 alive the 19-th time [Thu May 30 15:42:20 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> [Thu May 30 15:42:20 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" [Thu May 30 15:42:20 CST 2024] <<<<<< finish DM-080 pessimistic >>>>>> wait process dm-master1 exit... wait for rpc addr 127.0.0.1:8261 alive the 20-th time [Thu May 30 15:42:21 CST 2024] <<<<<< start DM-081 pessimistic >>>>>> start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" dmctl test cmd: "list-member" dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "resume-task test" run tidb sql failed 6-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 check alive timeout check data wait process dm-master1 exit... dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff successfully ============================== test_master_ha_when_enable_tidb_and_only_ca_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" check log contain failed 1-th time, retry later dmctl test cmd: "start-relay -s mysql-replica-02 worker2" [Pipeline] sh wait process dm-master1 exit... wait process dm-master.test exit... wait process dm-master.test exit... [Pipeline] sh dmctl test cmd: "resume-task test" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:42 . drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:42 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 30 15:42 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:42 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 .github -rw-r--r-- 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r-- 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r-- 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r-- 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 bin drwxr-sr-x 18 jenkins jenkins 4.0K May 30 15:42 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 30 15:42 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 30 15:42 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:42 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 30 15:42 engine -rwxr-xr-x 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 examples -rw-r--r-- 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r-- 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 metrics drwxr-sr-x 49 jenkins jenkins 4.0K May 30 15:42 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:42 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:42 tests -rw-r--r-- 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=4 expected=4 dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 7-th time, retry later + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:42 . drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:42 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 30 15:42 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:42 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 .github -rw-r--r-- 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r-- 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r-- 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r-- 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 bin drwxr-sr-x 18 jenkins jenkins 4.0K May 30 15:42 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 30 15:42 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 30 15:42 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:42 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 30 15:42 engine -rwxr-xr-x 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 examples -rw-r--r-- 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r-- 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 metrics drwxr-sr-x 49 jenkins jenkins 4.0K May 30 15:42 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:42 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:42 tests -rw-r--r-- 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break check diff successfully dmctl test cmd: "pause-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "resume-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" wait process dm-master1 exit... wait process dm-master.test exit... process dm-master.test already exit [Pipeline] { check diff successfully dmctl test cmd: "purge-relay --filename dm-it-4e2bdead-0cd8-4b7e-b714-66409f0e9164-q99bj-sd2n5-bin.000003 -s mysql-replica-01" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Pipeline] } [Pipeline] } wait process dm-master1 exit... process dm-master1 already exit [Thu May 30 15:42:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) try to isolate dm-worker1 kill dm-worker1 dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Thu May 30 15:42:24 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> [Thu May 30 15:42:24 CST 2024] <<<<<< start DM-LIST_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" [Pipeline] // timeout [Pipeline] // timeout [Pipeline] sh dmctl test cmd: "query-status test" got=1 expected=1 [Pipeline] sh wait process dm-worker.test exit... wait process dm-master.test exit... + '[' TLS_GROUP == G09 ']' + echo 'run G09 test' run G09 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ wait process tidb-server exit... process tidb-server already exit [Pipeline] checkout The recommended git tool is: git + make dm_integration_test_in_group GROUP=G09 + '[' TLS_GROUP == G10 ']' + echo 'run G10 test' run G10 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ wait process dm-master.test exit... start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt check diff failed 1-th time, retry later + make dm_integration_test_in_group GROUP=G10 run tidb sql failed 8-th time, retry later wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... wait process dm-master.test exit... /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt got=0 expected=1 command: query-status test Unsupported modify column: this column has primary key flag count: 0 != expected: 1, failed the 0-th time, will retry again check diff successfully read binlog from relay log failed, and will use remote binlog /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G09 Run cases: import_v10x sharding2 ha ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7b44cb6a; decorates RemoteLauncher[hudson.remoting.Channel@7a2b3ba7:JNLP4-connect connection from 10.233.97.238/10.233.97.238:47720] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 wait process dm-master.test exit... process dm-master.test already exit Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G10 Run cases: start_task print_status http_apis new_relay all_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:42:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> try to isolate dm-worker2 kill dm-worker2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker2 exit... run tidb sql failed 9-th time, retry later failpoint=github.com/pingcap/tiflow/dm/syncer/dbconn/GetMasterStatusFailed=return(1152) [Thu May 30 15:42:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... [Thu May 30 15:42:28 CST 2024] <<<<<< finish DM-081 pessimistic >>>>>> wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:42:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit [Thu May 30 15:42:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> dmctl test cmd: "pause-task test" got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:14435" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog list test" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [import_v10x] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit source1 master status, dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh-bin.000001, 1049, 09d84461-1e58-11ef-90ce-ca7cbfcff6e1:1-4 source2 master status, dm-it-e67b9dc7-337a-42c6-8478-6fc3505d79b9-986p0-qgzbh-bin.000001, 1158, 0a385bb0-1e58-11ef-bf81-ca7cbfcff6e1:1-4 [Thu May 30 15:42:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) got=3 expected=3 dmctl test cmd: "resume-task test" wait process dm-master.test exit... dmctl test cmd: "binlog inject test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:14627 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique (c);" dmctl test cmd: "query-status test" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] withEnv [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [start_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:42:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=3 expected=3 got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog skip test" [Thu May 30 15:42:29 CST 2024] <<<<<< start DM-082 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] container [Pipeline] { wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:42:29 CST 2024] <<<<<< test case dmctl_basic success! >>>>>> start running case: [dmctl_command] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit run tidb sql failed 1-th time, retry later [Pipeline] stage [Pipeline] { (Test) rpc addr 127.0.0.1:8262 is alive [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { run tidb sql failed 10-th time, retry later [Thu May 30 15:42:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/1/source1.yaml" start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" dmctl test cmd: "query-status test" wait process dm-master.test exit... got=2 expected=2 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:42:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] container [Pipeline] { dmctl test cmd: "stop-task test" [Pipeline] sh run tidb sql failed 11-th time, retry later [Thu May 30 15:42:32 CST 2024] <<<<<< finish DM-LIST_ERROR >>>>>> [Thu May 30 15:42:32 CST 2024] <<<<<< start DM-SKIP_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive exit code should be not zero exit code should be not zero [Thu May 30 15:42:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit got=3 expected=3 dmctl test cmd: "config task test --path /tmp/dm_test/import_v10x/task.yaml" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:42:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 12-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" [Thu May 30 15:42:33 CST 2024] <<<<<< finish DM-082 pessimistic >>>>>> + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 30 07:42 . drwxr-sr-x 17 1000 1000 4.0K May 30 07:42 .. -rw------- 1 root 1000 1.7K May 30 07:42 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 ca.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 client-cert.pem -rw------- 1 root 1000 1.7K May 30 07:42 client-key.pem -rw------- 1 root 1000 1.7K May 30 07:42 private_key.pem -rw-r--r-- 1 root 1000 451 May 30 07:42 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 30 07:42 server-cert.pem -rw------- 1 root 1000 1.7K May 30 07:42 server-key.pem dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" [Pipeline] } resume task will also initial failed dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/wrong-source.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml -w worker1" [Thu May 30 15:42:34 CST 2024] <<<<<< start DM-085 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" [Thu May 30 15:42:34 CST 2024] <<<<<< finish DM-SKIP_ERROR >>>>>> [Thu May 30 15:42:34 CST 2024] <<<<<< start DM-REPLACE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" dmctl test cmd: "list-member --name worker1" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w wrong-worker" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w worker1" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column new_col text, add column c int; alter table handle_error.tb1 add unique(c);" run tidb sql failed 13-th time, retry later run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-30 09:42:33'" dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml" dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" [Thu May 30 15:42:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... run tidb sql failed 14-th time, retry later dmctl test cmd: "query-status test" check diff successfully wait process dm-worker.test exit... check diff successfully check diff successfully dmctl test cmd: "validation status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "stop-task test" got=1 expected=1 got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 0\/0\/1" count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker2" [Thu May 30 15:42:38 CST 2024] <<<<<< finish DM-REPLACE_ERROR >>>>>> [Thu May 30 15:42:38 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING pessimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:42:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:42:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Thu May 30 15:42:38 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Thu May 30 15:42:38 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-30 09:42:33'" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Thu May 30 15:42:39 CST 2024] <<<<<< finish DM-085 pessimistic >>>>>> [Thu May 30 15:42:39 CST 2024] <<<<<< start DM-085 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Thu May 30 15:42:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '1995-03-07 01:02:03'" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:42:40 CST 2024] <<<<<< test case import_v10x success! >>>>>> start running case: [sharding2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:42:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 15-th time, retry later dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-only-ca.yaml" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2037-12-12 01:02:03'" [Thu May 30 15:42:41 CST 2024] <<<<<< finish DM-085 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm) 3049818624 bytes in 5.42 secs (562840265 bytes/sec) [Pipeline] { rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source1.yaml" [Thu May 30 15:42:42 CST 2024] <<<<<< finish DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> wait process dm-worker.test exit... [Thu May 30 15:42:42 CST 2024] <<<<<< start DM-086 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" dmctl test cmd: "stop-task test" [Thu May 30 15:42:42 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING pessimistic >>>>>> [Thu May 30 15:42:42 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetSessionVariableFailed=return("sql_mode,1152") wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-worker exit... process dm-worker already exit [Thu May 30 15:42:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Thu May 30 15:42:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { got=1 expected=1 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] sh [Thu May 30 15:42:44 CST 2024] <<<<<< finish DM-086 pessimistic >>>>>> [Thu May 30 15:42:44 CST 2024] <<<<<< start DM-086 optimistic >>>>>> { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-2.yaml --remove-meta=true" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit + pwd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow + ls -alh total 384K drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:42 . drwxr-sr-x 19 jenkins jenkins 4.0K May 30 15:42 .. -rw-r--r-- 1 jenkins jenkins 2.0K May 30 15:35 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 30 15:42 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 30 15:35 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:42 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 .github -rw-r--r-- 1 jenkins jenkins 985 May 30 15:35 .gitignore -rw-r--r-- 1 jenkins jenkins 2.9K May 30 15:35 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 30 15:35 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 30 15:35 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 30 15:35 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 30 15:35 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 30 15:35 Makefile.engine -rw-r--r-- 1 jenkins jenkins 724 May 30 15:35 OWNERS -rw-r--r-- 1 jenkins jenkins 215 May 30 15:35 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 30 15:35 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 30 15:35 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 30 15:35 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 30 15:35 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 bin drwxr-sr-x 18 jenkins jenkins 4.0K May 30 15:42 cdc drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 cdcv2 drwxr-sr-x 12 jenkins jenkins 4.0K May 30 15:42 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 30 15:42 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 30 15:42 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 30 15:42 engine -rwxr-xr-x 1 jenkins jenkins 26K May 30 15:35 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 examples -rw-r--r-- 1 jenkins jenkins 20K May 30 15:35 go.mod -rw-r--r-- 1 jenkins jenkins 155K May 30 15:35 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 30 15:42 metrics drwxr-sr-x 49 jenkins jenkins 4.0K May 30 15:42 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:42 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 30 15:42 tests -rw-r--r-- 1 jenkins jenkins 582 May 30 15:35 third-party-license.txt drwxr-sr-x 3 jenkins jenkins 4.0K May 30 15:42 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } [Thu May 30 15:42:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] // timeout rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:42:45 CST 2024] <<<<<< start DM-MULTIPLE_ROWS >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 [Pipeline] sh [Thu May 30 15:42:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source2.yaml" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... dmctl test cmd: "query-status test2" [Thu May 30 15:42:47 CST 2024] <<<<<< finish DM-086 optimistic >>>>>> wait process dm-worker.test exit... check data check log contain failed 1-th time, retry later dmctl test cmd: "start-relay -s mysql-replica-02 worker2" check diff successfully check diff successfully ============================== test_worker_handle_multi_tls_tasks success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive dmctl test cmd: "stop-task test" [Thu May 30 15:42:48 CST 2024] <<<<<< start DM-094 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:42:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:42:48 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING optimistic >>>>>> [Thu May 30 15:42:48 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/1/source1.yaml" wait process dm-master.test exit... query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit check log contain failed 2-th time, retry later got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" got=2 expected=2 check diff successfully check sync diff for the increment replication check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" check diff successfully dmctl test cmd: "resume-task test" wait process dm-master.test exit... + '[' TLS_GROUP == G11 ']' + echo 'run G11 test' run G11 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G11 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml" got=3 expected=3 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G11 rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:42:50 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive Run cases: adjust_gtid async_checkpoint_flush binlog_parse case_sensitive checkpoint_transaction check_task dm_syncer downstream_diff_index downstream_more_column drop_column_with_index duplicate_event expression_filter extend_column fake_rotate_event foreign_key full_mode gbk gtid ha_cases http_proxies lightning_load_task lightning_mode metrics new_collation_off only_dml openapi s3_dumpling_lightning sequence_sharding_optimistic sequence_sharding_removemeta shardddl_optimistic slow_relay_writer sql_mode sync_collation tracker_ignored_ddl validator_basic validator_checkpoint incompatible_ddl_changes ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) waiting for asynchronous relay and subtask to be started wait process dm-master.test exit... [Thu May 30 15:42:51 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" check sync diff for the increment replication check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait process dm-master.test exit... run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:42:52 CST 2024] <<<<<< finish DM-094 pessimistic >>>>>> [Thu May 30 15:42:52 CST 2024] <<<<<< start DM-094 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" [Thu May 30 15:42:52 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> [Thu May 30 15:42:52 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-master.test exit... dmctl test cmd: "query-status -s mysql-replica-01" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [adjust_gtid] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:42:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" dmctl test cmd: "query-status" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success check diff successfully kill dm-worker1 wait process dm-worker.test exit... wait process dm-master.test exit... got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" run tidb sql failed 2-th time, retry later got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:42:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" wait process dm-worker1 exit... wait process dm-worker.test exit... wait process dm-master.test exit... check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source1.yaml" got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:42:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:42:55 CST 2024] <<<<<< test case initial_unit success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // dir [Pipeline] } check diff successfully dmctl test cmd: "validation status test" [Pipeline] // withCredentials [Thu May 30 15:42:55 CST 2024] <<<<<< finish DM-094 optimistic >>>>>> [Pipeline] } wait process dm-master.test exit... [Pipeline] // timeout got=1 expected=1 got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 0\/0\/1" count: 0 != expected: 1, failed the 0-th time, will retry again [Pipeline] } [Pipeline] // stage [Pipeline] } run tidb sql failed 3-th time, retry later [Thu May 30 15:42:56 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } rpc addr 127.0.0.1:8262 is alive [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv dmctl test cmd: "stop-task test" [Thu May 30 15:42:56 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> [Thu May 30 15:42:56 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" [Pipeline] } wait process dm-worker2 exit... [Thu May 30 15:42:56 CST 2024] <<<<<< start DM-095 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" [Pipeline] // stage [Pipeline] } wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/adjust_gtid/source2.yaml" wait process dm-worker2 exit... process dm-worker2 already exit [Thu May 30 15:42:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again run tidb sql failed 4-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:42:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive wait for port 8262 offline the 1-th time got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Thu May 30 15:43:00 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP pessimistic >>>>>> [Thu May 30 15:43:00 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait for port 8262 offline the 2-th time check diff successfully dmctl test cmd: "validation status test" [Thu May 30 15:43:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=4 expected=4 got=1 expected=1 dmctl test cmd: "validation status --table-stage running test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again got=4 expected=4 got=0 expected=0 dmctl test cmd: "validation status --table-stage stopped test" got=2 expected=2 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=0 expected=1 command: validation show-error --error all test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again run tidb sql failed 5-th time, retry later dmctl test cmd: "pause-task test" [Thu May 30 15:43:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Thu May 30 15:43:03 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Thu May 30 15:43:03 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Thu May 30 15:42:53 CST 2024] <<<<<< finish test_kill_master_in_sync >>>>>> [Thu May 30 15:42:53 CST 2024] <<<<<< start test_kill_worker_in_sync >>>>>> [Thu May 30 15:42:53 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "resume-task test" got=3 expected=3 check diff failed 1-th time, retry later check diff successfully 39 11 11 66 dmctl test cmd: "stop-task test" [Thu May 30 15:43:04 CST 2024] <<<<<< finish DM-MULTIPLE_ROWS >>>>>> import prepare data start DM worker and master cluster [Thu May 30 15:43:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Thu May 30 15:43:04 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Thu May 30 15:43:04 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 102k 0 --:--:-- --:--:-- --:--:-- 104k rpc addr 127.0.0.1:8361 is alive got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" check diff successfully [Thu May 30 15:43:05 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP optimistic >>>>>> [Thu May 30 15:43:05 CST 2024] <<<<<< start DM-SKIP_INCOMPATIBLE_DDL >>>>>> dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8461 is alive [Thu May 30 15:43:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status --table-stage stopped" dmctl test cmd: "validation status --table-stage start test" dmctl test cmd: "validation show-error --error resolved test" dmctl test cmd: "validation show-error --error all" dmctl test cmd: "validation ignore-error test 100 --all" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "validation resolve-error test 100 --all" dmctl test cmd: "validation clear-error test 100 --all" got=1 expected=1 check diff successfully read binlog from relay log failed, and will use remote binlog wait process dm-worker exit... dmctl test cmd: "validation ignore-error test 100 101" dmctl test cmd: "validation resolve-error test 100 101" dmctl test cmd: "validation clear-error test 100 101" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Thu May 30 15:43:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "validation ignore-error test error-id" dmctl test cmd: "validation resolve-error test error-id" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" dmctl test cmd: "validation clear-error test error-id" dmctl test cmd: "validation ignore-error test" dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=2 check sync diff for the increment replication check diff successfully dmctl test cmd: "query-status test" dmctl test cmd: "validation resolve-error test" dmctl test cmd: "validation clear-error test" wait process dm-worker exit... process dm-worker already exit [Thu May 30 15:43:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Thu May 30 15:43:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh: line 556: hello: command not found dmctl test cmd: "query-status test" wait process dm-master.test exit... got=3 expected=3 got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "validation ignore-error" dmctl test cmd: "validation resolve-error" got=2 expected=2 got=1 expected=1 dmctl test cmd: "binlog skip test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 88709 0 --:--:-- --:--:-- --:--:-- 118k "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused dmctl test cmd: "validation clear-error" dmctl test cmd: "validation ignore-error non-exist-task-name 1" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:43:07 CST 2024] <<<<<< finish DM-095 pessimistic >>>>>> [Thu May 30 15:43:07 CST 2024] <<<<<< start DM-095 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" dmctl test cmd: "validation resolve-error non-exist-task-name 1" dmctl test cmd: "validation clear-error non-exist-task-name 1" [Thu May 30 15:43:07 CST 2024] <<<<<< finish DM-SKIP_INCOMPATIBLE_DDL >>>>>> [Thu May 30 15:43:07 CST 2024] <<<<<< start DM-4202 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:43:07 CST 2024] <<<<<< start DM-CAUSALITY >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "validation resolve-error test 1" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation show-error --error unprocessed test" wait process dm-worker.test exit... wait process dm-master.test exit... process dm-worker.test already exit process dm-master.test already exit [Thu May 30 15:43:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error test" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" [Thu May 30 15:43:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation show-error --error all test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test 2" dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit got=1 expected=1 dmctl test cmd: "validation clear-error test 1" dmctl test cmd: "validation status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "pause-task test" got=1 expected=1 dmctl test cmd: "validation show-error --error ignored test" got=1 expected=1 dmctl test cmd: "validation clear-error test 2" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:35724" dmctl test cmd: "query-status test" dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "validation status test" got=1 expected=1 dmctl test cmd: "validation stop test" [Thu May 30 15:43:09 CST 2024] <<<<<< finish DM-CAUSALITY >>>>>> [Thu May 30 15:43:09 CST 2024] <<<<<< start DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> got=2 expected=2 run tidb sql failed 1-th time, retry later got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" got=1 expected=1 dmctl test cmd: "validation status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=3 expected=3 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 60852 0 60852 0 0 16.5M 0 --:--:-- --:--:-- --:--:-- 19.3M % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 69287 0 69287 0 0 26.6M 0 --:--:-- --:--:-- --:--:-- 33.0M dmctl test cmd: "validation show-error --error all test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test 3" dmctl test cmd: "query-status test" [Thu May 30 15:43:09 CST 2024] <<<<<< finish DM-095 optimistic >>>>>> wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "validation resolve-error test 4" dmctl test cmd: "validation status test" command: query-status test "unit": "Sync" count: 0 != expected: 1 { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-01", "worker": "worker1", "result": null, "relayStatus": { "masterBinlog": "(dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g-bin.000001, 10630)", "masterBinlogGtid": "d8ba5909-1e57-11ef-9145-7a3b0554a828:1-45", "relaySubDir": "d8ba5909-1e57-11ef-9145-7a3b0554a828.000001", "relayBinlog": "(dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g-bin.000001, 10630)", "relayBinlogGtid": "d8ba5909-1e57-11ef-9145-7a3b0554a828:1-45", "relayCatchUpMaster": true, "stage": "Running", "result": null } }, "subTaskStatus": [ { "name": "test", "stage": "Running", "unit": "Load", "result": null, "unresolvedDDLLockID": "", "load": { "finishedBytes": "0", "totalBytes": "0", "progress": "0.00 %", "metaBinlog": "(dm-it-f3295d73-95eb-4647-9f64-cf9b5b0e9a13-jzx5v-l1s5g-bin.000001, 10630)", "metaBinlogGTID": "d8ba5909-1e57-11ef-9145-7a3b0554a828:1-45", "bps": "0" }, "validation": null } ] } ] } PASS coverage: 3.3% of statements in github.com/pingcap/tiflow/dm/... got=1 expected=1 dmctl test cmd: "validation clear-error test --all" dmctl test cmd: "validation status test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused start DM task got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "validation status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " got=2 expected=2 dmctl test cmd: "query-status test" [Thu May 30 15:43:10 CST 2024] <<<<<< start DM-096 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchTargetDoTablesFailed=return(1152) got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "stop-task test" clean up data dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/dm-task.yaml --remove-meta" dmctl test cmd: "stop-task test" [Thu May 30 15:43:11 CST 2024] <<<<<< finish DM-4202 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:11 CST 2024] <<<<<< test case sharding2 success! >>>>>> start running case: [ha] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM worker and master [Thu May 30 15:43:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master1.toml >>>>>> [Thu May 30 15:43:11 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:43:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader dmctl test cmd: "query-status test" [Thu May 30 15:43:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase [Thu May 30 15:43:12 CST 2024] <<<<<< start DM-4204 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" check diff successfully dmctl test cmd: "pause-task test" check data check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/adjust_gtid/dm-task.yaml" dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 118 100 118 0 0 163k 0 --:--:-- --:--:-- --:--:-- 115k dmctl test cmd: "validation status test" got=2 expected=2 --> validation resolve-error --all 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=3 expected=3 check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:43:13 CST 2024] <<<<<< finish DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> [Thu May 30 15:43:13 CST 2024] <<<<<< start DM-UpdateBARule optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait for port 8262 offline the 1-th time start dm-worker2 and kill dm-worker1 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/0/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Thu May 30 15:43:14 CST 2024] <<<<<< finish DM-4204 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker1 exit... check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" [Thu May 30 15:43:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" use sync_diff_inspector to check increment data check diff successfully [Thu May 30 15:43:15 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source kill dm-worker1 got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/shardddl1/task.yaml" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Thu May 30 15:43:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... [Thu May 30 15:43:15 CST 2024] <<<<<< start DM-4206 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/adjust_gtid/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker1 exit... process dm-worker1 already exit mysql_ssl_setup at=/var/lib/mysql/ [Thu May 30 15:43:16 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/tls/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:43:16 CST 2024] <<<<<< finish DM-096 pessimistic >>>>>> [Thu May 30 15:43:16 CST 2024] <<<<<< start DM-096 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker1 exit... rpc addr 127.0.0.1:8262 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha/source1.yaml" wait process dm-worker.test exit... dmctl test cmd: "query-status test" dmctl test cmd: "show-ddl-locks" got=1 expected=1 restart dm-master got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker1 exit... process dm-worker1 already exit start worker3 [Thu May 30 15:43:17 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:39469 -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:39276 -s mysql-replica-01" rpc addr 127.0.0.1:8263 is alive check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 [Thu May 30 15:43:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time resume task to worker2 dmctl test cmd: "resume-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "pause-task test" dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:36408 -s mysql-replica-02" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:36627 -s mysql-replica-02" {"connections":0,"version":"8.0.11-TiDB-v7.5.1-164-g39ea2b30d3","git_hash":"39ea2b30d32ab8cf486f30ca318cf2e4bd99eaef"}dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task-2.yaml " wait process dm-master exit... dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker3" [Thu May 30 15:43:18 CST 2024] <<<<<< finish DM-4206 pessimistic >>>>>> [Thu May 30 15:43:18 CST 2024] <<<<<< start DM-4206 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:43:19 CST 2024] <<<<<< finish DM-096 optimistic >>>>>> check diff successfully ============================== test_worker_ha_when_enable_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master exit... process dm-master already exit wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 kill dm-worker2 [Thu May 30 15:43:20 CST 2024] <<<<<< start DM-097 pessimistic >>>>>> run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:41438 -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:41245 -s mysql-replica-01" [Thu May 30 15:43:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:38566 -s mysql-replica-02" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit [Thu May 30 15:43:21 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:38785 -s mysql-replica-02" dmctl test cmd: "query-status test" wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 [Thu May 30 15:43:21 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully got=3 expected=3 run tidb sql failed 1-th time, retry later run tidb sql failed 2-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker2 exit... process dm-worker2 already exit start worker4 [Thu May 30 15:43:21 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:22 CST 2024] <<<<<< test case relay_interrupt success! >>>>>> start running case: [safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh... Verbose mode = false wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:22 CST 2024] <<<<<< test case adjust_gtid success! >>>>>> start running case: [async_checkpoint_flush] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" rpc addr 127.0.0.1:18262 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive query-status from all dm-master dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8461 is alive ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 1-th time, got value: '' wait process dm-master exit... got=3 expected=3 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 3-th time wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test2" wait for rpc addr 127.0.0.1:8261 alive the 2-th time run tidb sql failed 3-th time, retry later ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 2-th time, got value: '' wait process dm-master exit... process dm-master already exit [Thu May 30 15:43:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "stop-task test" [Thu May 30 15:43:24 CST 2024] <<<<<< finish DM-4206 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 4-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 3-th time, got value: '' rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:43:25 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... check diff successfully dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/async_checkpoint_flush/source1.yaml" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait for rpc addr 127.0.0.1:8261 alive the 5-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/conf/dm-task.yaml " wait process dm-master.test exit... metric dm_master_worker_state{worker="worker1"} has invalid value start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml " run tidb sql failed 4-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:43214" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:40724" check diff successfully dmctl test cmd: "query-status test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" check diff successfully PID of insert_data is 1988 got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" [Thu May 30 15:43:27 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Thu May 30 15:43:27 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 6-th time [Thu May 30 15:43:27 CST 2024] <<<<<< finish DM-097 pessimistic >>>>>> [Thu May 30 15:43:27 CST 2024] <<<<<< start DM-097 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again run tidb sql failed 5-th time, retry later wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "show-ddl-locks" dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:44990" dmctl test cmd: "query-status test" wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:42663" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" start dm-worker3 and kill dm-worker2 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 [Thu May 30 15:43:30 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> [Thu May 30 15:43:30 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" run tidb sql failed 6-th time, retry later check diff successfully dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " run tidb sql failed 2-th time, retry later wait process dm-master exit... wait process dm-worker2 exit... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' [Thu May 30 15:43:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master-no-tls.toml >>>>>> dmctl test cmd: "validation show-error test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:47024" wait process dm-master exit... process dm-master already exit [Thu May 30 15:43:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit [Thu May 30 15:43:32 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:44602" run tidb sql failed 7-th time, retry later wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Thu May 30 15:43:32 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Thu May 30 15:43:32 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait for rpc addr 127.0.0.1:8261 alive the 2-th time run tidb sql failed 3-th time, retry later got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:33 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /tmp/dm_test/tls/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time run tidb sql failed 8-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation resolve-error test --all" dmctl test cmd: "validation show-error test" got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 --> validation ignore-error --all 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:49058" dmctl test cmd: "query-status test" wait process dm-worker.test exit... wait process dm-worker.test exit... check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:46541" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-no-tls.yaml" got=3 expected=3 run tidb sql failed 1-th time, retry later [Thu May 30 15:43:35 CST 2024] <<<<<< finish DM-097 optimistic >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "show-ddl-locks" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... [Thu May 30 15:43:35 CST 2024] <<<<<< finish DM-UpdateBARule optimistic >>>>>> [Thu May 30 15:43:35 CST 2024] <<<<<< start DM-RENAME_TABLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 start task and check stage wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:36 CST 2024] <<<<<< start DM-098 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-no-tls.yaml --remove-meta=true" run tidb sql failed 9-th time, retry later start to run safe mode case 1 [Thu May 30 15:43:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchAllDoTablesFailed=return(1152) dmctl test cmd: "query-status test" got=2 expected=2 [Thu May 30 15:43:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later dmctl test cmd: "stop-task test" [Thu May 30 15:43:37 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 got=1 expected=1 check data check diff successfully ============================== test_source_and_target_with_empty_tlsconfig success ================================== [Thu May 30 15:43:38 CST 2024] <<<<<< start DM-4209 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" run tidb sql failed 10-th time, retry later wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker1 exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" [Thu May 30 15:43:39 CST 2024] <<<<<< finish DM-RENAME_TABLE pessimistic >>>>>> [Thu May 30 15:43:39 CST 2024] <<<<<< start DM-RENAME_TABLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:43:40 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" run tidb sql failed 11-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" wait for rpc addr 127.0.0.1:8261 alive the 2-th time start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait for the task to be scheduled and keep paused HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive resume task before kill and restart dm-worker dmctl test cmd: "resume-task test" [Thu May 30 15:43:41 CST 2024] <<<<<< finish DM-098 pessimistic >>>>>> [Thu May 30 15:43:41 CST 2024] <<<<<< finish DM-RENAME_TABLE optimistic >>>>>> [Thu May 30 15:43:41 CST 2024] <<<<<< start DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time start dm-worker2 and kill dm-worker3 dmctl test cmd: "stop-task test" Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 104 wait process dm-worker.test exit... [Thu May 30 15:43:42 CST 2024] <<<<<< finish DM-4209 pessimistic >>>>>> [Thu May 30 15:43:42 CST 2024] <<<<<< start DM-4209 optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-worker3 exit... rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" check diff successfully wait process dm-worker3 exit... process dm-worker3 already exit [Thu May 30 15:43:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit [Thu May 30 15:43:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:43:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:46 CST 2024] <<<<<< test case tls success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) wait process dm-worker.test exit... [Pipeline] // cache dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" [Pipeline] } got=3 expected=3 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "stop-task test" [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:43:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] } wait process dm-worker.test exit... [Thu May 30 15:43:47 CST 2024] <<<<<< finish DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:47 CST 2024] <<<<<< test case shardddl3 success! >>>>>> start running case: [shardddl3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh... Verbose mode = false [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master exit... dmctl test cmd: "stop-task test" [Thu May 30 15:43:48 CST 2024] <<<<<< finish DM-4209 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive got=2 expected=2 wait process dm-master exit... process dm-master already exit [Thu May 30 15:43:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:49 CST 2024] <<<<<< test case many_tables success! >>>>>> start running case: [online_ddl] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:43:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:43:49 CST 2024] <<<<<< start DM-4211 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... check diff successfully dmctl test cmd: "validation status test" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:43:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test --binlog-pos dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:54386 alter table handle_error.tb1 add column c int;" success success [Thu May 30 15:43:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:43:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source1.yaml" [Thu May 30 15:43:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:43:51 CST 2024] <<<<<< finish DM-4211 >>>>>> got=2 expected=2 [Thu May 30 15:43:51 CST 2024] <<<<<< start DM-RemoveLock pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source1.yaml" dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/online_ddl/check-task.yaml" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 [Thu May 30 15:43:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 1-th time, will retry again rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source2.yaml" [Thu May 30 15:43:52 CST 2024] <<<<<< start DM-4213 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/online_ddl/source2.yaml" dmctl test cmd: "query-status test" wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Thu May 30 15:43:54 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" wait for rpc addr 127.0.0.1:8264 alive the 1-th time [Thu May 30 15:43:53 CST 2024] <<<<<< start DM-099 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 [Thu May 30 15:43:54 CST 2024] <<<<<< finish DM-4213 >>>>>> dmctl test cmd: "start-task /tmp/dm_test/online_ddl/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker3" dmctl test cmd: "query-status test" check log contain failed 1-th time, retry later dmctl test cmd: "list-member --name worker3" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=2 expected=2 got=1 expected=1 query-status from all dm-master dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" check sync diff after restart DDL owner dmctl test cmd: "validation show-error test" got=3 expected=3 join new dm-master and query-status [Thu May 30 15:43:56 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master4.toml >>>>>> use sync_diff_inspector to check full dump data check diff successfully alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 got=0 expected=1 command: validation show-error test "id": "1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8561 alive the 1-th time wait process dm-worker2 exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker2 exit... check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "query-status test" got=3 expected=3 process dm-worker2 already exit [Thu May 30 15:43:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:43:58 CST 2024] <<<<<< finish DM-RemoveLock pessimistic >>>>>> [Thu May 30 15:43:58 CST 2024] <<<<<< start DM-RemoveLock optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation ignore-error test --all" dmctl test cmd: "validation show-error test" got=0 expected=0 dmctl test cmd: "validation status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:43:59 CST 2024] <<<<<< finish DM-099 pessimistic >>>>>> check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/async_checkpoint_flush/run.sh: line 29: 1988 Terminated insert_data 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 success check log contain failed 1-th time, retry later wait process dm-master.test exit... success finish running run safe mode recover case 1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:00 CST 2024] <<<<<< start test_COMMIT_in_QueryEvent >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/start_task/1/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... [Thu May 30 15:44:00 CST 2024] <<<<<< start DM-100 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:00 CST 2024] <<<<<< test case handle_error success! >>>>>> start running case: [handle_error_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/start_task/1/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:02 CST 2024] <<<<<< test case async_checkpoint_flush success! >>>>>> start running case: [binlog_parse] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker1 exit... wait process dm-worker.test exit... [Thu May 30 15:44:02 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time wait process dm-worker.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; 1.1.0 rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... got=3 expected=3 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source1.yaml" wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:44:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 2 [Thu May 30 15:44:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> alter: add column c int comment '1 2 3๐Ÿ˜Š4'; rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time pt-online-schema-change 3.0.12 ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. [Thu May 30 15:44:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:44:05 CST 2024] <<<<<< finish DM-100 optimistic >>>>>> use sync_diff_inspector to check increment data check diff successfully [Thu May 30 15:43:59 CST 2024] <<<<<< finish test_isolate_master_and_worker >>>>>> 5 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:44:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:44:06 CST 2024] <<<<<< start DM-101 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/start_task/1/dm-task.yaml --remove-meta=true" wait for rpc addr 127.0.0.1:8261 alive the 2-th time ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source1.yaml" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source2.yaml" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:44:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" [Thu May 30 15:44:07 CST 2024] <<<<<< start DM-4215 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 check diff successfully rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/binlog_parse/source1.yaml" prepare data start task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/binlog_parse/conf/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:08 CST 2024] <<<<<< test case ha_cases3_1 success! >>>>>> start running case: [ha_master] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit start DM worker and master [Thu May 30 15:44:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Thu May 30 15:44:09 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> [Thu May 30 15:44:09 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> [Thu May 30 15:44:09 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> [Thu May 30 15:44:09 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 wait process dm-worker2 exit... [Thu May 30 15:44:09 CST 2024] <<<<<< START DM-MASTER on port 8761, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master6.toml >>>>>> wait for rpc addr 127.0.0.1:8761 alive the 1-th time dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" check full phase check diff successfully dmctl test cmd: "validation start test" prepare incremental data check incremental phase dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8761 is alive dmctl test cmd: "query-status test" got=2 expected=2 --> (fail) validation start: missing mode value dmctl test cmd: "validation start --mode" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive check diff successfully <<<<<< test_COMMIT_in_QueryEvent success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=3 expected=3 ******************************************************************* Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER possibly with SSL_ca_file|SSL_ca_path for verification. If you really don't want to verify the certificate and keep the connection open to Man-In-The-Middle attacks please set SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application. ******************************************************************* at /usr/bin/pt-online-schema-change line 6976. dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" --> (fail) validation start: invalid mode dmctl test cmd: "validation start --mode xxx" --> (fail) validation start: missing start-time value dmctl test cmd: "validation start --start-time" wait process dm-worker2 exit... process dm-worker2 already exit [Thu May 30 15:44:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8561 is alive rpc addr 127.0.0.1:8661 is alive --> (fail) validation start: invalid start-time dmctl test cmd: "validation start --start-time xx" wait process dm-master.test exit... --> (fail) validation start: without all-task and task-name dmctl test cmd: "validation start --mode full" --> (fail) validation start: with both all-task and task-name dmctl test cmd: "validation start --all-task test" --> (fail) validation start: too many arguments dmctl test cmd: "validation start test test2" got=2 expected=2 got=1 expected=1 got=1 expected=1 prepare incremental data 2 dmctl test cmd: "validation start test" check incremental phase 2 check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "stop-task test" --> (fail) validation start: non-existed subtask dmctl test cmd: "validation start xxxxx" --> (fail) validation start: non-exist source dmctl test cmd: "validation start -s xxxxx --all-task" alter: add column c int comment '1 2 3๐Ÿ˜Š4'; pt-online-schema-change 3.0.12 check diff successfully [Thu May 30 15:44:12 CST 2024] <<<<<< finish DM-101 optimistic >>>>>> --> (fail) validation stop: without all-task and task-name dmctl test cmd: "validation stop" [Thu May 30 15:44:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time --> (fail) validation stop: with both all-task and task-name dmctl test cmd: "validation stop --all-task test" --> (fail) validation stop: too many arguments dmctl test cmd: "validation stop test test2" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit --> (fail) validation stop: non-existed subtask dmctl test cmd: "validation stop xxxxx" --> (fail) validation stop: non-exist source dmctl test cmd: "validation stop -s xxxxx --all-task" wait process dm-worker.test exit... [Thu May 30 15:44:13 CST 2024] <<<<<< start DM-102 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" --> (fail) validation stop: stop not-enabled validator dmctl test cmd: "validation stop -s mysql-replica-01 test" wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... --> (success) validation start: start validation without explicit mode for source 1 dmctl test cmd: "validation start -s mysql-replica-01 test" rpc addr 127.0.0.1:8263 is alive use sync_diff_inspector to check full data check diff successfully alter: add column age int; 1.1.0 wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:44:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time --> (fail) validation start: start all validator with explicit mode, but 1 subtask already enabled dmctl test cmd: "validation start --mode full test" --> (fail) validation start: start validation with explicit mode for source 1 again dmctl test cmd: "validation start -s mysql-replica-01 --mode full test" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 --> (fail) validation start: start all validator without explicit mode dmctl test cmd: "validation start test" wait process dm-worker.test exit... --> (fail) validation stop: stop all but 1 subtask is not enabled dmctl test cmd: "validation stop test" --> (success) validation start: start validation with fast mode and start-time for source 2 dmctl test cmd: "validation start -s mysql-replica-02 --mode fast --start-time '2020-01-01 00:01:00' test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:15 CST 2024] <<<<<< test case start_task success! >>>>>> start running case: [print_status] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh... Verbose mode = false +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status ++ pwd + cur=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status + source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../_utils/test_prepare ++ source env_variables +++ MYSQL_HOST1=127.0.0.1 +++ MYSQL_HOST2=127.0.0.1 +++ TIDB_HOST=127.0.0.1 +++ MYSQL_PORT1=3306 +++ MYSQL_PORT2=3307 +++ MYSQL_PASSWORD1=123456 +++ MYSQL_PASSWORD2=123456 +++ TIDB_PASSWORD=123456 +++ TIDB_PORT=4000 +++ MASTER_PORT=8261 +++ MASTER_PEER_PORT=8291 +++ MASTER_PORT1=8261 +++ MASTER_PORT2=8361 +++ MASTER_PORT3=8461 +++ MASTER_PORT4=8561 +++ MASTER_PORT5=8661 +++ MASTER_PORT6=8761 +++ MASTER_PEER_PORT1=8291 +++ MASTER_PEER_PORT2=8292 +++ MASTER_PEER_PORT3=8293 +++ MASTER_PEER_PORT4=8294 +++ MASTER_PEER_PORT5=8295 +++ MASTER_PEER_PORT6=8296 +++ WORKER1_PORT=8262 +++ WORKER2_PORT=8263 +++ WORKER3_PORT=8264 +++ WORKER4_PORT=18262 +++ WORKER5_PORT=18263 +++ WORKER1_NAME=worker1 +++ WORKER2_NAME=worker2 +++ SOURCE_ID1=mysql-replica-01 +++ SOURCE_ID2=mysql-replica-02 +++ RESET_MASTER=true +++ VERBOSE=false ++ '[' true = true ']' ++ run_sql 'RESET MASTER' 3306 123456 ++ run_sql 'RESET MASTER' 3307 123456 + WORK_DIR=/tmp/dm_test/print_status + cleanup_data print_status + rm -rf /tmp/dm_test/print_status + mkdir /tmp/dm_test/print_status + for target_db in '"$@"' + run_sql 'drop database if exists `print_status`' 4000 123456 + run_sql 'drop database if exists dm_meta' 4000 123456 + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=0 + echo '0 dm-master alive' 0 dm-master alive + pkill -hup dm-master.test + true ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=0 + echo '0 dm-worker alive' 0 dm-worker alive + pkill -hup dm-worker.test + true ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test process dm-master.test already exit + wait_process_exit dm-worker.test process dm-worker.test already exit rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:44:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "validation status test" + wait_process_exit dm-syncer.test process dm-syncer.test already exit + run + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.prepare.sql 127.0.0.1 3306 123456 + inject_points=("github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)" "github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)" "github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)") ++ join_string ';' 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)' 'github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)' 'github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' ++ local 'IFS=;' ++ shift ++ echo 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + GO_FAILPOINTS='github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-worker1.toml /tmp/dm_test/print_status/dm-worker1.toml + sed -i s%placeholder%/tmp/dm_test/print_status/relay_by_worker%g /tmp/dm_test/print_status/dm-worker1.toml + run_dm_master /tmp/dm_test/print_status/master 8261 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml [Thu May 30 15:44:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_master_online 127.0.0.1:8261 wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation start: start all validator of the task without explicit param again, i.e. resuming dmctl test cmd: "validation start test" wait process dm-worker.test exit... kill dm-master1 dmctl test cmd: "validation status test" dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb`" kill dm-master1 and kill dm-master2 alter: add column age int; 1.1.0 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation stop: stop validation of source 1 dmctl test cmd: "validation stop -s mysql-replica-01 test" rpc addr 127.0.0.1:8261 is alive + run_dm_worker /tmp/dm_test/print_status/worker1 8262 /tmp/dm_test/print_status/dm-worker1.toml [Thu May 30 15:44:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/print_status/dm-worker1.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_worker_online 127.0.0.1:8262 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "validation status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:16 CST 2024] <<<<<< test case binlog_parse success! >>>>>> start running case: [case_sensitive] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:16 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master1 exit... dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" wait process dm-master1 exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 --> (success) validation stop: stop all of test dmctl test cmd: "validation stop test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation stop: stop all of test again dmctl test cmd: "validation stop test" [Thu May 30 15:44:17 CST 2024] <<<<<< finish DM-102 >>>>>> rpc addr 127.0.0.1:8262 is alive + cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/source1.yaml /tmp/dm_test/print_status/source1.yaml + sed -i '/relay-binlog-name/i\relay-dir: /tmp/dm_test/print_status/worker1/relay_log' /tmp/dm_test/print_status/source1.yaml + dmctl_operate_source create /tmp/dm_test/print_status/source1.yaml mysql-replica-01 + operate=create + source_toml=/tmp/dm_test/print_status/source1.yaml + source_id=mysql-replica-01 + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'operate-source create /tmp/dm_test/print_status/source1.yaml' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "operate-source create /tmp/dm_test/print_status/source1.yaml" wait process dm-master1 exit... process dm-master1 already exit kill dm-master2 wait process dm-master1 exit... process dm-master1 already exit + dmctl_start_task_standalone + '[' 0 -ge 2 ']' + remove_meta= + '[' 0 -ge 1 ']' + task_conf=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml ' '"result": true' 2 '"source": "mysql-replica-01"' 1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml " dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: start all of test dmctl test cmd: "validation start test" dmctl test cmd: "validation status test" alter: add key name (name); 1.1.0 wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 --> (success) validation stop: multiple source dmctl test cmd: "validation stop -s mysql-replica-01 -s mysql-replica-02 test" [Thu May 30 15:44:18 CST 2024] <<<<<< start DM-103 pessimistic >>>>>> dmctl test cmd: "validation status test" wait process dm-master2 exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=2 expected=2 --> (success) validation start: multiple source dmctl test cmd: "validation start -s mysql-replica-01 -s mysql-replica-02 test" dmctl test cmd: "validation status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=2 expected=2 got=0 expected=0 dmctl test cmd: "stop-task test" wait process dm-master2 exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task-no-validator.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit success success [Thu May 30 15:44:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:44:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time alter: add key name (name); 1.1.0 wait process dm-master2 exit... process dm-master2 already exit initial cluster of dm-masters have been killed now we will check whether joined masters can work normally dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master2 exit... process dm-master2 already exit master5 [Thu May 30 15:44:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2-no-validator.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 wait for rpc addr 127.0.0.1:8262 alive the 2-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check diff successfully + ls /tmp/dm_test/print_status/relay_by_worker/worker1/0930e629-1e58-11ef-bb11-42dd8a299ecf.000001 /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index /tmp/dm_test/print_status/relay_by_worker/worker1/0930e629-1e58-11ef-bb11-42dd8a299ecf.000001: dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0-bin.000001 relay.meta + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.increment.sql 127.0.0.1 3306 123456 + check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml check diff failed 1-th time, retry later wait for rpc addr 127.0.0.1:8262 alive the 3-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml" alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:44:22 CST 2024] <<<<<< finish DM-103 pessimistic >>>>>> [Thu May 30 15:44:22 CST 2024] <<<<<< start DM-103 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" [Thu May 30 15:44:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=1 expected=1 --> (fail) validation update: on non-existed validator dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid b7b934ab-1e57-11ef-9cf0-f2dfe2c7a753:1-999" --> (success) validation start: start all tasks dmctl test cmd: "validation start --all-task --mode fast" wait for rpc addr 127.0.0.1:8262 alive the 4-th time --> (fail) validation update: missing task-name dmctl test cmd: "validation update" --> (fail) validation update: multi tasks dmctl test cmd: "validation update test test2" --> (fail) validation update: non-existed subtask dmctl test cmd: "validation update xxxxx" use sync_diff_inspector to check increment2 data now! check diff successfully 4 dm-master alive 3 dm-worker alive 0 dm-syncer alive --> (fail) validation update: non-exist source dmctl test cmd: "validation update -s xxxxx test" check diff successfully + check_log_contains /tmp/dm_test/print_status/worker1/log/dm-worker.log 'enable safe-mode because of task initialization.*duration=1m0s' + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=1 + echo '1 dm-master alive' 1 dm-master alive + pkill -hup dm-master.test ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=1 + echo '1 dm-worker alive' 1 dm-worker alive + pkill -hup dm-worker.test ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test --> (fail) validation update: not specify cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-02" --> (fail) validation update: not specify cutover-binlog-pos dmctl test cmd: "validation update test2 -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" wait for rpc addr 127.0.0.1:8262 alive the 5-th time alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 --> (fail) validation update: invalid cutover-binlog-pos dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos xxx" --> (fail) validation update: specify both cutover-binlog-pos and cutover-binlog-gtid dmctl test cmd: "validation update test -s mysql-replica-01 --cutover-binlog-pos '(mysql-bin.000001, 2345)' --cutover-binlog-gtid b7b934ab-1e57-11ef-9cf0-f2dfe2c7a753:1-999" check diff failed 1-th time, retry later --> (success) validation update: on exist source dmctl test cmd: "validation update test -s mysql-replica-02 --cutover-binlog-gtid b7b934ab-1e57-11ef-9cf0-f2dfe2c7a753:1-999" dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=0 got=2 expected=2 got=2 expected=2 got=0 expected=0 got=2 expected=2 got=1 expected=1 dmctl test cmd: "validation status test2" check sync diff after restart DDL owner dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" got=1 expected=1 got=0 expected=0 got=1 expected=1 got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 --> (success) validation stop: stop all tasks dmctl test cmd: "validation stop --all-task" dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=0 got=2 expected=2 got=0 expected=0 got=2 expected=2 dmctl test cmd: "validation status test2" wait for rpc addr 127.0.0.1:8262 alive the 6-th time wait process dm-master.test exit... got=1 expected=1 got=0 expected=0 got=1 expected=1 got=0 expected=0 got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit + wait_process_exit dm-worker.test wait for rpc addr 127.0.0.1:8262 alive the 7-th time check diff successfully wait process dm-master.test exit... wait process dm-master.test exit... wait pattern dm-worker1.toml exit... alter: add column age int; 1.1.0 check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:44:26 CST 2024] <<<<<< finish DM-103 optimistic >>>>>> wait pattern dm-worker1.toml exit... wait process dm-worker.test exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:44:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... [Thu May 30 15:44:27 CST 2024] <<<<<< start DM-104 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Thu May 30 15:44:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 alter: add column age int; 1.1.0 success success finish running run safe mode recover case 2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8263 alive the 2-th time wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + check_print_status + i=0 + '[' 0 -lt 3 ']' ++ grep 'dm-worker exit' /tmp/dm_test/print_status/worker1/log/dm-worker.log + exit_log='[2024/05/30 15:44:26.847 +08:00] [INFO] [main.go:90] ["dm-worker exit"]' + '[' '[2024/05/30 15:44:26.847 +08:00] [INFO] [main.go:90] ["dm-worker exit"]' == 'not found' ']' + break + '[' 0 -ge 3 ']' + echo 'checking print status' checking print status + dump_status_file=/tmp/dm_test/print_status/worker1/log/dump_status.log + grep -o 'progress status of dumpling' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=1 + '[' 1 -ge 1 ']' + grep -o '\[bps=0' /tmp/dm_test/print_status/worker1/log/dm-worker.log + true ++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log ++ awk '{print $1}' + dump_status_count=0 + '[' 0 -eq 0 ']' + status_file=/tmp/dm_test/print_status/worker1/log/loader_status.log + grep -oP '\[unit=lightning-load\] \[IsCanceled=false\] \[finished_bytes=59674\] \[total_bytes=59674\] \[progress=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/loader_status.log ++ awk '{print $1}' + status_count=1 + '[' 1 -eq 1 ']' + grep 'current speed (bytes / seconds)' /tmp/dm_test/print_status/worker1/log/dm-worker.log + grep -vq '"current speed (bytes / seconds)"=0' + echo 'check load unit print status success' check load unit print status success + status_file2=/tmp/dm_test/print_status/worker1/log/syncer_status.log + grep -oP '\[total_rows=[0-9]+\] \[total_rps=[0-9]+\] \[rps=[0-9]+\] \[master_position=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log ++ wc -l /tmp/dm_test/print_status/worker1/log/syncer_status.log ++ awk '{print $1}' + status_count2=13 + '[' 13 -ge 1 ']' + echo 'check sync unit print status success' check sync unit print status success ++ date + echo '[Thu May 30 15:44:28 CST 2024] <<<<<< test case print_status success! >>>>>>' [Thu May 30 15:44:28 CST 2024] <<<<<< test case print_status success! >>>>>> start running case: [http_apis] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait pattern dm-worker2.toml exit... got=2 expected=2 wait for rpc addr 127.0.0.1:8263 alive the 3-th time wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait process dm-master.test exit... run tidb sql failed 1-th time, retry later [Thu May 30 15:44:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait pattern dm-worker2.toml exit... wait process dm-master.test exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait for rpc addr 127.0.0.1:8263 alive the 4-th time wait process dm-master.test exit... wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Thu May 30 15:44:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit alter: add key name (name); 1.1.0 wait for rpc addr 127.0.0.1:8263 alive the 5-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-task test" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Thu May 30 15:44:32 CST 2024] <<<<<< finish DM-104 optimistic >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 6-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:32 CST 2024] <<<<<< test case ha success! >>>>>> wait process dm-worker.test exit... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // cache [Pipeline] } alter: add key name (name); 1.1.0 [Thu May 30 15:44:33 CST 2024] <<<<<< start DM-105 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate wait for rpc addr 127.0.0.1:8263 alive the 7-th time [Pipeline] } dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later [Pipeline] // withEnv wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Pipeline] } rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:44:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Pipeline] // stage [Pipeline] } start to run safe mode case 3 [Thu May 30 15:44:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 [Thu May 30 15:44:34 CST 2024] <<<<<< finish DM-4215 pessimistic >>>>>> [Thu May 30 15:44:34 CST 2024] <<<<<< start DM-4215 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive start dm-master1 and dm-master2 again [Thu May 30 15:44:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Thu May 30 15:44:34 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time run tidb sql failed 1-th time, retry later alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 wait for rpc addr 127.0.0.1:8262 alive the 2-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:44:34 CST 2024] <<<<<< finish DM-RemoveLock optimistic >>>>>> wait process dm-master exit... wait process dm-master exit... process dm-master already exit [Thu May 30 15:44:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" rpc addr 127.0.0.1:8262 is alive source-id: mysql-replica-01\nflavor: ''\nenable-gtid: true\nenable-relay: true\nrelay-dir: /tmp/dm_test/http_apis/worker1/relay_log\nrelay-binlog-name: ''\nrelay-binlog-gtid: ''\nfrom:\n host: 127.0.0.1\n user: root\n password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=\n port: 3306\n % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed dmctl test cmd: "query-status test" got=1 expected=2 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 470 100 166 100 304 1290 2362 --:--:-- --:--:-- --:--:-- 2356 100 470 100 166 100 304 1289 2360 --:--:-- --:--:-- --:--:-- 2356 start task and check stage command: query-status test "ErrScope": "downstream" count: 1 != expected: 2, failed the 0-th time, will retry again ---\nname: test\ntask-mode: all\nis-sharding: false\nmeta-schema: \"dm_meta\"\n# enable-heartbeat: true\n\ntarget-database:\n host: \"127.0.0.1\"\n port: 4000\n user: \"root\"\n password: \"\"\n\nmysql-instances:\n - source-id: \"mysql-replica-01\"\n block-allow-list: \"instance\"\n mydumper-config-name: \"global\"\n loader-config-name: \"global\"\n syncer-config-name: \"global\"\n\nblock-allow-list:\n instance:\n do-dbs: [\"http_apis\"]\n\nmydumpers:\n global:\n threads: 4\n chunk-filesize: 0\n skip-tz-utc: true\n statement-size: 100\n extra-args: \"\"\n\nloaders:\n global:\n pool-size: 16\n dir: \"./dumped_data\"\n\nsyncers:\n global:\n worker-count: 16\n batch: 100\n HTTP 127.0.0.1:8261/apis/v1alpha1/status/test-task is alive % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed dmctl test cmd: "stop-task test" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:44:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "check-task /tmp/dm_test/dmctl_command/temp.yaml" [Thu May 30 15:44:37 CST 2024] <<<<<< finish DM-105 optimistic >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 742 0 0 100 742 0 740 0:00:01 0:00:01 --:--:-- 740 100 950 100 208 100 742 191 682 0:00:01 0:00:01 --:--:-- 682 dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; 1.1.0 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" [Thu May 30 15:44:38 CST 2024] <<<<<< start DM-106 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 278k 0 --:--:-- --:--:-- --:--:-- 301k get sub task configs % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2299 0 2299 0 0 540k 0 --:--:-- --:--:-- --:--:-- 561k pause task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 194 100 183 100 11 16268 977 --:--:-- --:--:-- --:--:-- 16636 dmctl test cmd: "list-member --master" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" got=2 expected=2 [Thu May 30 15:44:38 CST 2024] <<<<<< start DM-RestartMaster pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" got=5 expected=5 [Thu May 30 15:44:38 CST 2024] <<<<<< start test_evict_leader >>>>>> leader is master5 dmctl test cmd: "operate-leader evict" [Thu May 30 15:44:39 CST 2024] <<<<<< finish DM-4215 optimistic >>>>>> % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 13988 0 --:--:-- --:--:-- --:--:-- 16500 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 new leader is master4 wait process dm-master.test exit... process dm-master.test already exit alter: add column age int; pt-online-schema-change 3.0.12 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1960 100 1960 0 0 340k 0 --:--:-- --:--:-- --:--:-- 382k resume task and check stage % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 195 100 184 100 11 9336 558 --:--:-- --:--:-- --:--:-- 9684 run tidb sql failed 1-th time, retry later leader is master4 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 15774 0 --:--:-- --:--:-- --:--:-- 16500 alter: add column age int; new leader is master2 pt-online-schema-change 3.0.12 leader is master2 dmctl test cmd: "operate-leader evict" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later wait process dm-worker.test exit... alter: add key name (name); % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 14024 0 --:--:-- --:--:-- --:--:-- 16500 new leader is master1 [Thu May 30 15:44:40 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1854 100 1854 0 0 190k 0 --:--:-- --:--:-- --:--:-- 201k pt-online-schema-change 3.0.12 leader is master1 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 17609 0 --:--:-- --:--:-- --:--:-- 33000 alter: add key name (name); pt-online-schema-change 3.0.12 new leader is master3 check diff successfully leader is master3 dmctl test cmd: "operate-leader evict" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 13801 0 --:--:-- --:--:-- --:--:-- 16500 cancel evict leader on master1, and master1 will be the leader dmctl test cmd: "operate-leader cancel-evict" wait process dm-worker.test exit... dmctl test cmd: "query-status test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 801 100 801 0 0 111k 0 --:--:-- --:--:-- --:--:-- 130k alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; pt-online-schema-change 3.0.12 cancel evict leader twice, and test cancel evict leader from http interface % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 18867 0 --:--:-- --:--:-- --:--:-- 33000 leader is master1 cancel evict leader on all masters cancel master port 8361 dmctl test cmd: "operate-leader cancel-evict" dmctl test cmd: "stop-task test" got=2 expected=2 wait process dm-worker.test exit... [Thu May 30 15:44:42 CST 2024] <<<<<< finish DM-106 optimistic >>>>>> alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; pt-online-schema-change 3.0.12 cancel master port 8461 dmctl test cmd: "operate-leader cancel-evict" cancel master port 8561 dmctl test cmd: "query-status test" dmctl test cmd: "operate-leader cancel-evict" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:42 CST 2024] <<<<<< test case dmctl_command success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) cancel master port 8661 dmctl test cmd: "operate-leader cancel-evict" [Thu May 30 15:44:42 CST 2024] <<<<<< finish test_evict_leader >>>>>> [Thu May 30 15:44:42 CST 2024] <<<<<< start test_list_member_command >>>>>> got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:5043 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:5548 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" alter: add column age int; pt-online-schema-change 3.0.12 [Pipeline] // cache [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master [Thu May 30 15:44:43 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Thu May 30 15:44:43 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> [Pipeline] // dir kill dm-worker1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" current leader is master1 dmctl test cmd: "list-member --master" alter: add column age int; pt-online-schema-change 3.0.12 [Pipeline] } wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:44:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:44:43 CST 2024] <<<<<< start DM-107 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" got=5 expected=5 kill leader master1 [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } alter: add key name (name); pt-online-schema-change 3.0.12 wait process dm-master exit... [Pipeline] // container after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive check diff failed 1-th time, retry later [Pipeline] } wait process dm-worker1 exit... [Pipeline] // withEnv [Pipeline] } alter: add key name (name); [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate pt-online-schema-change 3.0.12 check diff successfully wait process master1 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:44:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master exit... process dm-master already exit dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:6850 alter table handle_error.tb1 add column c varchar(20);" pt-online-schema-change 3.0.12 wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --worker --name=worker1" got=1 expected=1 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 33 100 33 0 0 8126 0 --:--:-- --:--:-- --:--:-- 11000 check data [Pipeline] } got=3 expected=3 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:7521 alter table handle_error.tb1 add column c varchar(20);" check diff successfully 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process master1 exit... process master1 already exit alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL; pt-online-schema-change 3.0.12 [Thu May 30 15:44:45 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> [Thu May 30 15:44:45 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Paused count: 0 != expected: 1, failed the 0-th time, will retry again use sync_diff_inspector to check increment data check diff successfully start dm-worker3 and kill dm-worker2 wait process dm-master.test exit... [Pipeline] // withEnv current leader is master2 dmctl test cmd: "list-member --master" got=4 expected=4 kill leader master2 wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:47 CST 2024] <<<<<< test case http_apis success! >>>>>> start running case: [new_relay] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/run.sh... Verbose mode = false dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:44:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... wait process master2 exit... dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:8915 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" [Pipeline] } got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:9494 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:44:48 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Thu May 30 15:44:48 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker2 exit... process dm-worker2 already exit [Thu May 30 15:44:48 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process master2 exit... got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" [Thu May 30 15:44:48 CST 2024] <<<<<< finish DM-107 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time process master2 already exit [Thu May 30 15:44:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] // stage dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" [Pipeline] } dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8264 is alive wait and check task running dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" wait process dm-master.test exit... process dm-master.test already exit [Thu May 30 15:44:49 CST 2024] <<<<<< start DM-108 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:10980 alter table handle_error.tb1 add column c varchar(20);" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:49 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:11467 alter table handle_error.tb1 add column c varchar(20);" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Thu May 30 15:44:50 CST 2024] <<<<<< finish DM-RestartMaster pessimistic >>>>>> [Thu May 30 15:44:50 CST 2024] <<<<<< start DM-RestartMaster optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8361 is alive [Thu May 30 15:44:50 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit success success [Thu May 30 15:44:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:44:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time use sync_diff_inspector to check increment2 data now! check diff successfully [Thu May 30 15:44:51 CST 2024] <<<<<< start DM-4219 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-master.toml >>>>>> check diff successfully check log contain failed 1-th time, retry later current leader is master4 dmctl test cmd: "list-member --master" got=5 expected=5 wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 check log contain failed 1-th time, retry later current leader is master4 kill master1 wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:44:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" wait process dm-master1 exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" check diff failed 2-th time, retry later run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" check log contain failed 2-th time, retry later got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" got=1 expected=1 got=2 expected=2 got=1 expected=1 restart dm-master wait process dm-master1 exit... process dm-master1 already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source1.yaml" [Thu May 30 15:44:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... check diff failed 3-th time, retry later dmctl test cmd: "stop-task test" wait process dm-master exit... [Thu May 30 15:44:55 CST 2024] <<<<<< finish DM-4219 pessimistic >>>>>> [Thu May 30 15:44:55 CST 2024] <<<<<< start DM-4219 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/case_sensitive/source2.yaml" wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully wait process dm-master exit... process dm-master already exit check sync diff after restart DDL owner check log contain failed 3-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/case_sensitive/dm-task.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:44:56 CST 2024] <<<<<< test case online_ddl success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } check diff failed at last dmctl test cmd: "stop-task test" [Pipeline] // stage [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Thu May 30 15:44:57 CST 2024] <<<<<< finish DM-108 optimistic >>>>>> [Pipeline] // node [Pipeline] } got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" run tidb sql failed 1-th time, retry later check diff successfully [Thu May 30 15:44:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait pattern dm-worker1.toml exit... check log contain failed 4-th time, retry later wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... [Thu May 30 15:44:58 CST 2024] <<<<<< start DM-109 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... [Thu May 30 15:44:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Thu May 30 15:44:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Thu May 30 15:44:57 CST 2024] <<<<<< finish test_kill_worker_in_sync >>>>>> 3 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... got=2 expected=2 success dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:45:00 CST 2024] <<<<<< finish DM-4219 optimistic >>>>>> success finish running run safe mode recover case 3 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check log contain failed 5-th time, retry later check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "shard-ddl-lock" wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member --master" got=5 expected=5 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 wait pattern dm-worker2.toml exit... current leader is master4 kill master2 [Thu May 30 15:45:01 CST 2024] <<<<<< finish DM-RestartMaster optimistic >>>>>> [Thu May 30 15:45:01 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait pattern dm-worker2.toml exit... [Thu May 30 15:45:01 CST 2024] <<<<<< start DM-4220 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait process dm-master.test exit... wait pattern dm-worker2.toml exit... wait process dm-master2 exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Thu May 30 15:45:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/case_sensitive/conf/dm-worker2.toml >>>>>> >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_leak passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master2 exit... process dm-master2 already exit dmctl test cmd: "query-status test" got=1 expected=2 command: query-status test Unsupported modify column: this column has primary key flag count: 1 != expected: 2, failed the 0-th time, will retry again wait process dm-master.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-master.test exit... check diff failed 3-th time, retry later check diff successfully check diff failed 1-th time, retry later got=2 expected=2 dmctl test cmd: "binlog revert test" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 4 [Thu May 30 15:45:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:16594" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:17605" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:16787" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:17824" dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:16787" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:17824" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" check diff failed at last dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:06 CST 2024] <<<<<< test case ha_cases_2 success! >>>>>> start running case: [ha_cases2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:06 CST 2024] <<<<<< start test_pause_task >>>>>> [Thu May 30 15:45:06 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" [Thu May 30 15:45:07 CST 2024] <<<<<< finish DM-109 optimistic >>>>>> check diff successfully rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time run tidb sql failed 1-th time, retry later check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:45:08 CST 2024] <<<<<< start DM-110 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:45:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:45:08 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> wait for rpc addr 127.0.0.1:8361 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8361 is alive dmctl test cmd: "list-member --master" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" check diff successfully dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:16787" dmctl test cmd: "query-status test" got=2 expected=2 import prepare data start DM worker and master [Thu May 30 15:45:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>> [Thu May 30 15:45:09 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>> [Thu May 30 15:45:09 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=5 expected=5 dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" check diff failed 1-th time, retry later dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:17824" dmctl test cmd: "binlog revert test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:17824" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:45:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time current leader is master4 kill master3 dmctl test cmd: "binlog revert test" check diff failed 1-th time, retry later dmctl test cmd: "stop-task test" [Thu May 30 15:45:10 CST 2024] <<<<<< finish DM-4220 pessimistic >>>>>> [Thu May 30 15:45:10 CST 2024] <<<<<< start DM-4220 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:45:10 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait process dm-master3 exit... wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check diff failed 2-th time, retry later rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" got=2 expected=2 dmctl test cmd: "binlog revert test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master3 exit... process dm-master3 already exit check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:45:12 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS pessimistic >>>>>> [Thu May 30 15:45:12 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-relay -s mysql-replica-01 worker1 worker2" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:19218" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:20496" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Thu May 30 15:45:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>> check diff successfully dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:19411" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:20715" wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:19411" got=3 expected=3 dmctl test cmd: "transfer-source mysql-replica-01 worker1" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" after restart dm-worker, task should resume automatically HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:20715" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... check diff failed 1-th time, retry later run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml" check diff failed 3-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:45:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully check diff failed 1-th time, retry later [Thu May 30 15:45:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:45:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully check diff failed 1-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" [Thu May 30 15:45:16 CST 2024] <<<<<< finish DM-110 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml" dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:20715" dmctl test cmd: "binlog revert test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:20715" dmctl test cmd: "binlog revert test" dmctl test cmd: "stop-task test" [Thu May 30 15:45:16 CST 2024] <<<<<< finish DM-4220 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" [Thu May 30 15:45:16 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time got=2 expected=2 [Thu May 30 15:45:17 CST 2024] <<<<<< start DM-111 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check diff successfully kill dm-worker1 [Thu May 30 15:45:17 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time [Thu May 30 15:45:17 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" check diff successfully check diff successfully rpc addr 127.0.0.1:8461 is alive dmctl test cmd: "list-member --master" got=5 expected=5 dmctl test cmd: "stop-task test" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/case_sensitive/source2.yaml" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" current leader is master4 wait process dm-worker.test exit... wait process dm-worker1 exit... current leader is master4 kill master5 rpc addr 127.0.0.1:8264 is alive [Thu May 30 15:45:18 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> [Thu May 30 15:45:18 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS optimistic >>>>>> [Thu May 30 15:45:18 CST 2024] <<<<<< start DM-COLUMN_INDEX pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:18262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit success dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=2 expected=2 check diff successfully [Thu May 30 15:45:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait process dm-master.test exit... wait process dm-master5 exit... got=2 expected=2 success [Thu May 30 15:45:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:45:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:18262 is alive [Thu May 30 15:45:19 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:21842" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:23387" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process dm-master5 exit... process dm-master5 already exit got=3 expected=3 dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later [Thu May 30 15:45:21 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Thu May 30 15:45:21 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml " got=3 expected=3 check diff failed 2-th time, retry later wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "query-status -s mysql-replica-01" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:22 CST 2024] <<<<<< test case case_sensitive success! >>>>>> dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:23811" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase start running case: [checkpoint_transaction] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:25545" process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:45:23 CST 2024] <<<<<< finish DM-COLUMN_INDEX pessimistic >>>>>> [Thu May 30 15:45:23 CST 2024] <<<<<< start DM-COLUMN_INDEX optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "purge-relay --filename dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0-bin.000005 -s mysql-replica-01" kill dm-worker1 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" check diff failed 3-th time, retry later [Thu May 30 15:45:24 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> [Thu May 30 15:45:24 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-worker1.toml >>>>>> wait process dm-worker1 exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/source1.yaml" [Thu May 30 15:45:25 CST 2024] <<<<<< finish DM-COLUMN_INDEX optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-task.yaml " check diff failed at last dmctl test cmd: "stop-task test" got=2 expected=2 [Thu May 30 15:45:25 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time [Thu May 30 15:45:25 CST 2024] <<<<<< finish DM-111 optimistic >>>>>> dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:26038" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:27703" dmctl test cmd: "query-status test" wait process dm-worker exit... wait process dm-worker2 exit... got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later [Thu May 30 15:45:26 CST 2024] <<<<<< start DM-112 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check sync diff after restart DDL owner check diff successfully rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "list-member --master" got=5 expected=5 dmctl test cmd: "list-member --worker --name=worker1,worker2" wait process dm-worker exit... wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source1.yaml" process dm-worker already exit [Thu May 30 15:45:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Thu May 30 15:45:27 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 1-th time, retry later dmctl test cmd: "list-member --worker" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "list-member" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=2 expected=2 dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:45:28 CST 2024] <<<<<< start DM-DML_EXECUTE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" check diff failed 1-th time, retry later kill dm-master1 dmctl test cmd: "stop-task test" [Thu May 30 15:45:29 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Thu May 30 15:45:29 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" use sync_diff_inspector to check increment data check diff successfully check diff successfully [Thu May 30 15:45:28 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "config export -p /tmp/configs" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... got=2 expected=2 success check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 success finish running run safe mode recover case 4 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "list-member --name=worker1,worker2" got=2 expected=2 kill worker1 got=2 expected=2 start dumping SQLs into source pause tasks test dmctl test cmd: "pause-task test" dmctl test cmd: "pause-task test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:28265" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:29861" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" pause tasks test2 dmctl test cmd: "pause-task test2" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later dmctl test cmd: "pause-task test2" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test2" wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:45:32 CST 2024] <<<<<< finish DM-DML_EXECUTE_ERROR >>>>>> wait process dm-worker.test exit... start check_worker_ungraceful_stop_with_retry times: 0 wait process dm-master.test exit... process dm-master.test already exit check diff failed 3-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit dmctl test cmd: "list-member --name=worker1,worker2" got=1 expected=1 got=1 expected=1 kill worker2 resume tasks test dmctl test cmd: "resume-task test" wait process dm-worker exit... got=3 expected=3 dmctl test cmd: "resume-task test" wait process dm-worker.test exit... start check_worker_ungraceful_stop_with_retry times: 1 wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "query-status test" resume tasks test2 dmctl test cmd: "resume-task test2" dmctl test cmd: "stop-task test" [Thu May 30 15:45:34 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> got=3 expected=3 dmctl test cmd: "resume-task test2" got=3 expected=3 dmctl test cmd: "query-status test2" wait process dm-worker2 exit... wait process dm-worker exit... process dm-worker already exit [Thu May 30 15:45:34 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Thu May 30 15:45:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... check diff failed at last dmctl test cmd: "stop-task test" [Thu May 30 15:45:35 CST 2024] <<<<<< start DM-4201 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start check_worker_ungraceful_stop_with_retry times: 2 check_worker_ungraceful_stop_with_retry success after retry: 2 after ungraceful exit data in source count: 31 data in tidb count: 17 ungraceful stop test success [Thu May 30 15:45:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:45:35 CST 2024] <<<<<< finish DM-112 optimistic >>>>>> wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "list-member" got=2 expected=2 [Thu May 30 15:45:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:45:35 CST 2024] <<<<<< start DM-KEY_NOT_FOUND >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:36 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:45:36 CST 2024] <<<<<< start DM-113 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:45:36 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:30234" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" got=2 expected=2 check diff successfully check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:45:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:45:37 CST 2024] <<<<<< finish DM-4201 >>>>>> rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" [Thu May 30 15:45:37 CST 2024] <<<<<< finish DM-KEY_NOT_FOUND >>>>>> got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config master master1" check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "config master master2" got=1 expected=1 [Thu May 30 15:45:38 CST 2024] <<<<<< finish test_list_member_command >>>>>> wait process dm-worker exit... dmctl test cmd: "start-relay -s mysql-replica-01 worker1" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "config import -p /tmp/configs" wait process dm-worker exit... process dm-worker already exit [Thu May 30 15:45:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Thu May 30 15:45:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" got=2 expected=2 start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-task.yaml --remove-meta" [Thu May 30 15:45:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:45:40 CST 2024] <<<<<< finish DM-113 pessimistic >>>>>> [Thu May 30 15:45:40 CST 2024] <<<<<< start DM-113 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Thu May 30 15:45:40 CST 2024] <<<<<< start DM-001 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file kill dm-master1 and kill dm-master2 wait process dm-worker.test exit... dmctl test cmd: "operate-source show" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 check diff successfully >>>>>>>>>>>>>>>>>>>>>>>>>>test test_relay_operations passed 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" wait process dm-master1 exit... got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... check diff failed 2-th time, retry later [Thu May 30 15:45:42 CST 2024] <<<<<< finish DM-001 >>>>>> [Thu May 30 15:45:42 CST 2024] <<<<<< finish DM-113 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task-safe-mode-duration.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master1 exit... process dm-master1 already exit wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:43 CST 2024] <<<<<< test case handle_error_2 success! >>>>>> start running case: [handle_error_3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh... Verbose mode = false [Thu May 30 15:45:43 CST 2024] <<<<<< start DM-002 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" [Thu May 30 15:45:43 CST 2024] <<<<<< start DM-114 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully wait process dm-master2 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 3-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:45:45 CST 2024] <<<<<< finish DM-002 >>>>>> wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-master2 exit... process dm-master2 already exit wait and check task running HTTP 127.0.0.1:8461/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source1.yaml" [Thu May 30 15:45:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:45:46 CST 2024] <<<<<< start DM-003 pessimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit [Thu May 30 15:45:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:45:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 4-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source2.yaml" rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later [Thu May 30 15:45:47 CST 2024] <<<<<< finish DM-114 pessimistic >>>>>> [Thu May 30 15:45:47 CST 2024] <<<<<< start DM-114 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=3 expected=3 dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " use sync_diff_inspector to check data now! check diff successfully dmctl test cmd: "offline-member --master --name master1" dmctl test cmd: "offline-member --master --name master2" kill dm-master3 [Thu May 30 15:45:48 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" check diff failed 5-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master3 exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml " got=2 expected=2 [Thu May 30 15:45:50 CST 2024] <<<<<< finish DM-114 optimistic >>>>>> dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:1559 alter table handle_error.tb1 add column d varchar(20);" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-master3 exit... process dm-master3 already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:1735 alter table handle_error.tb1 add column d varchar(20);" check diff failed 6-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" [Thu May 30 15:45:51 CST 2024] <<<<<< start DM-115 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Thu May 30 15:45:51 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Thu May 30 15:45:51 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" check diff successfully finish running safe mode duration case 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=2 expected=2 kill dm-worker1 check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 [Thu May 30 15:45:52 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... wait process dm-worker1 exit... check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 7-th time, retry later wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:3856 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:4247 alter table handle_error.tb1 add column d varchar(20);" wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:45:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" rpc addr 127.0.0.1:8461 is alive check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Thu May 30 15:45:54 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> [Thu May 30 15:45:54 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_upstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff successfully pause task and check status dmctl test cmd: "pause-task test" check diff failed 1-th time, retry later wait process dm-worker.test exit... check diff failed at last check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Thu May 30 15:45:56 CST 2024] <<<<<< finish DM-003 pessimistic >>>>>> got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:6411 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:6759 alter table handle_error.tb1 add column d varchar(20);" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:45:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:45:57 CST 2024] <<<<<< start DM-004 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Thu May 30 15:45:57 CST 2024] <<<<<< finish DM-115 optimistic >>>>>> [Thu May 30 15:45:57 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Thu May 30 15:45:57 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... [Thu May 30 15:45:58 CST 2024] <<<<<< start DM-116 pessimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" [Thu May 30 15:45:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master-join1.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:45:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:8966 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:9271 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "pause-task test" got=3 expected=3 got=1 expected=1 got=1 expected=1 4 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" check diff failed 1-th time, retry later got=3 expected=3 dmctl test cmd: "stop-task test" [Thu May 30 15:46:00 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:46:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:01 CST 2024] <<<<<< finish DM-004 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" [Thu May 30 15:46:02 CST 2024] <<<<<< start DM-005 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta" [Thu May 30 15:46:01 CST 2024] <<<<<< start DM-4210 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source1.yaml" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" [Thu May 30 15:46:03 CST 2024] <<<<<< finish DM-116 pessimistic >>>>>> [Thu May 30 15:46:03 CST 2024] <<<<<< start DM-116 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" dmctl test cmd: "query-status test" wait process dm-master.test exit... got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 [Thu May 30 15:46:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:46:03 CST 2024] <<<<<< finish DM-005 >>>>>> wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:11239 alter table handle_error.tb1 add column d int;alter table handle_error.tb1 add unique(d);" dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column e int unique;" dmctl test cmd: "query-status test" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/source2.yaml" [Thu May 30 15:46:05 CST 2024] <<<<<< finish DM-116 optimistic >>>>>> got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "start-relay -s mysql-replica-02 worker2" [Thu May 30 15:46:06 CST 2024] <<<<<< start DM-117 optimistic >>>>>> check diff successfully wait for port 8262 offline the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait for port 8262 offline the 2-th time wait process dm-master.test exit... dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=2 got=1 expected=1 [Thu May 30 15:46:07 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl3_1/schema.sql -s mysql-replica-01" dmctl test cmd: "stop-task test" [Thu May 30 15:46:07 CST 2024] <<<<<< finish DM-4210 >>>>>> wait process dm-worker.test exit... wait process dm-master.test exit... [Thu May 30 15:46:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:46:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3" check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:46:08 CST 2024] <<<<<< finish DM-117 optimistic >>>>>> [Thu May 30 15:46:08 CST 2024] <<<<<< start DM-4193 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:09 CST 2024] <<<<<< test case ha_master success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } rpc addr 127.0.0.1:8262 is alive [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } dmctl test cmd: "query-status -s mysql-replica-02" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member -n worker3" [Pipeline] // container [Thu May 30 15:46:09 CST 2024] <<<<<< start DM-118 pessimistic >>>>>> rpc addr 127.0.0.1:8263 is alive [Pipeline] } [Pipeline] // withEnv [Pipeline] } dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:13027" wait process dm-worker.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:12152" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:11 CST 2024] <<<<<< test case shardddl1 success! >>>>>> start running case: [shardddl1_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh... Verbose mode = false got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:13027" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:12152" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:13413" dmctl test cmd: "query-status test" got=2 expected=2 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "binlog revert test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:13413" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:13413" check diff failed 1-th time, retry later dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:12590" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker1.toml >>>>>> got=3 expected=3 run tidb sql failed 1-th time, retry later wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" graceful pause test success start check pause diff check diff successfully resume task and check status dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" kill dm-worker1 wait process dm-worker1 exit... wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:46:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/checkpoint_transaction/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit [Thu May 30 15:46:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> [Thu May 30 15:46:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 stop task dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:14 CST 2024] <<<<<< finish DM-118 pessimistic >>>>>> [Thu May 30 15:46:14 CST 2024] <<<<<< start DM-118 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check sync diff after set SafeModeInitPhaseSeconds failpoint check diff successfully wait for one of DM-worker offine failed, retry later dmctl test cmd: "stop-task test" [Thu May 30 15:46:15 CST 2024] <<<<<< finish DM-4193 pessimistic >>>>>> [Thu May 30 15:46:15 CST 2024] <<<<<< start DM-4193 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for one of DM-worker offine failed, retry later [Thu May 30 15:46:16 CST 2024] <<<<<< start DM-SAME_DDL_TWICE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" [Thu May 30 15:46:16 CST 2024] <<<<<< finish DM-118 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> [Thu May 30 15:46:17 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker3.toml >>>>>> dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:15189" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:14529" wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:15189" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive DM-worker1 is sharding lock owner and detects it offline [Thu May 30 15:46:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:14529" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:15575" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:15575" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:15575" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8264 is alive [Thu May 30 15:46:18 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker2.toml >>>>>> dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:14967" dmctl test cmd: "query-status test" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive check diff failed 1-th time, retry later got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 dmctl test cmd: "query-status -s mysql-replica-02" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "list-member --worker" wait process dm-master.test exit... process dm-master.test already exit >>>>>>>>>>>>>>>>>>>>>>>>>>test test_restart_relay_status passed 1 dm-master alive 3 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "stop-task test" [Thu May 30 15:46:20 CST 2024] <<<<<< finish DM-SAME_DDL_TWICE pessimistic >>>>>> [Thu May 30 15:46:20 CST 2024] <<<<<< start DM-BINARY_COLUMN >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-sharding.yaml --remove-meta" [Thu May 30 15:46:21 CST 2024] <<<<<< finish DM-4193 optimistic >>>>>> wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully check diff successfully check diff successfully [Thu May 30 15:46:22 CST 2024] <<<<<< start DM-4230 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:22 CST 2024] <<<<<< test case shardddl3_1 success! >>>>>> start running case: [shardddl4] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-master.toml >>>>>> check diff successfully check diff successfully check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... [Thu May 30 15:46:23 CST 2024] <<<<<< finish DM-BINARY_COLUMN >>>>>> [Thu May 30 15:46:23 CST 2024] <<<<<< start DM-006 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column d int unique;" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog revert test" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Thu May 30 15:46:24 CST 2024] <<<<<< finish DM-4230 >>>>>> wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source1.yaml" [Thu May 30 15:46:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:46:25 CST 2024] <<<<<< start DM-4177 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source2.yaml" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:27 CST 2024] <<<<<< finish DM-006 >>>>>> got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" dmctl test cmd: "binlog skip wrong-task" dmctl test cmd: "binlog skip" dmctl test cmd: "binlog skip test -b dm-it-b9aeedc2-3031-4cbc-ad13-63bd6652420f-76drj-ddxf3-bin|000001.000001:19397 -s wrong-source" [Thu May 30 15:46:27 CST 2024] <<<<<< start DM-119 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog skip test -b mysql-bin|1111 -s wrong-source" dmctl test cmd: "query-status test" [Thu May 30 15:46:28 CST 2024] <<<<<< start DM-007 >>>>>> got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog skip test" wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int;"" dmctl test cmd: "binlog skip test -b mysql-bin.000000:00000" dmctl test cmd: "binlog aaa test" dmctl test cmd: "stop-task test" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" [Thu May 30 15:46:28 CST 2024] <<<<<< finish DM-4177 >>>>>> dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int" dmctl test cmd: "binlog revert test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:28 CST 2024] <<<<<< test case safe_mode success! >>>>>> start running case: [sequence_safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check diff failed 1-th time, retry later [Thu May 30 15:46:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Thu May 30 15:46:30 CST 2024] <<<<<< start DM-4231 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time kill dm-worker1 check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:46:31 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> got=1 expected=1 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int; alter table handle_error.tb1 add column d int unique;"" dmctl test cmd: "query-status test" wait process dm-worker1 exit... wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:32 CST 2024] <<<<<< finish DM-007 >>>>>> graceful stop test success start check stop diff check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit [Thu May 30 15:46:32 CST 2024] <<<<<< finish DM-4231 >>>>>> wait process dm-worker.test exit... wait process dm-worker1 exit... process dm-worker1 already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source1.yaml" [Thu May 30 15:46:33 CST 2024] <<<<<< start DM-008 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff failed 3-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process tidb-server exit... process tidb-server already exit [Thu May 30 15:46:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:34 CST 2024] <<<<<< test case checkpoint_transaction success! >>>>>> start running case: [check_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-master.test exit... check diff failed 4-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/dm-worker1.toml >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-task.yaml --remove-meta" wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/source1.yaml" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' >>>>>>>>>>>>>>>>>>>>>>>>>>test test_cant_dail_downstream passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:37 CST 2024] <<<<<< finish DM-008 >>>>>> wait process dm-worker.test exit... check diff failed 5-th time, retry later check diff successfully dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-noshard.yaml" wait for port 8262 offline the 1-th time [Thu May 30 15:46:38 CST 2024] <<<<<< start DM-009 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-sharding.yaml" --> start test_privileges_can_migrate... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:38 CST 2024] <<<<<< test case handle_error_3 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } wait for port 8262 offline the 2-th time [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } wait process dm-master.test exit... process dm-master.test already exit [Pipeline] // timeout [Pipeline] } dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // stage [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // container [Pipeline] } [Thu May 30 15:46:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // node check diff failed 6-th time, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Thu May 30 15:46:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-worker.test exit... check diff successfully dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" pass test_privileges_can_migrate --> start test_privilege_precheck... dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" wait process dm-worker.test exit... dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/check_task/conf/task-priv.yaml" pass test_privilege_precheck 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive check diff failed 7-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:41 CST 2024] <<<<<< finish DM-009 >>>>>> [Thu May 30 15:46:42 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... [Thu May 30 15:46:42 CST 2024] <<<<<< start DM-010 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check sync diff after reset failpoint check diff successfully rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/new_relay/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit check diff failed 8-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/new_relay/source1.yaml" check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit restart DM-worker after set SequenceShardSyncedExecutionExit failpoint [Thu May 30 15:46:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:46:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-worker.test exit... check diff failed 9-th time, retry later rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-01" got=2 expected=2 got=1 expected=1 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start running case: [dm_syncer] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-master.toml >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:46 CST 2024] <<<<<< finish DM-010 >>>>>> dmctl test cmd: "query-status -s mysql-replica-01" got=0 expected=1 command: query-status -s mysql-replica-01 "relayCatchUpMaster": true count: 0 != expected: 1, failed the 0-th time, will retry again wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:46:47 CST 2024] <<<<<< start DM-011 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed 10-th time, retry later wait for one of DM-worker offine failed, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:46:49 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/dm_syncer/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 >>>>>>>>>>>>>>>>>>>>>>>>>>test test_kill_dump_connection passed 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Thu May 30 15:46:49 CST 2024] <<<<<< finish DM-011 >>>>>> check diff failed at last dmctl test cmd: "stop-task test" wait for one of DM-worker offine failed, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dm_syncer/source1.yaml" [Thu May 30 15:46:50 CST 2024] <<<<<< finish DM-119 pessimistic >>>>>> [Thu May 30 15:46:50 CST 2024] <<<<<< start DM-119 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/dm_syncer/source2.yaml" [Thu May 30 15:46:50 CST 2024] <<<<<< start DM-012 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" DM-worker1 is sharding lock owner and detects it offline [Thu May 30 15:46:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" dmctl test cmd: "start-task /tmp/dm_test/dm_syncer/dm-task.yaml --remove-meta" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:46:52 CST 2024] <<<<<< finish DM-012 >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:53 CST 2024] <<<<<< start DM-013 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff failed 2-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:54 CST 2024] <<<<<< test case new_relay success! >>>>>> start running case: [all_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:46:54 CST 2024] <<<<<< start test_expression_filter >>>>>> [Thu May 30 15:46:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:46:55 CST 2024] <<<<<< finish DM-013 >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/bin/dm-syncer.test --log-file=/tmp/dm_test/dm_syncer/syncer1/log/dm-syncer.log --config=/tmp/dm_test/dm_syncer/dm-syncer-1.toml >> /tmp/dm_test/dm_syncer/syncer1/log/stdout.log 2>&1 & /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/bin/dm-syncer.test --log-file=/tmp/dm_test/dm_syncer/syncer2/log/dm-syncer.log --config=/tmp/dm_test/dm_syncer/dm-syncer-2.toml --meta=/tmp/dm_test/dm_syncer/old_meta_file --syncer-config-format >> /tmp/dm_test/dm_syncer/syncer2/log/stdout.log 2>&1 & wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff failed 3-th time, retry later [Thu May 30 15:46:56 CST 2024] <<<<<< start DM-014 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:46:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check sync diff after restart DDL owner check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:46:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 4-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" dmctl test cmd: "stop-task test" [Thu May 30 15:47:00 CST 2024] <<<<<< finish DM-014 >>>>>> wait process dm-worker.test exit... check diff failed 5-th time, retry later [Thu May 30 15:47:01 CST 2024] <<<<<< start DM-015 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... check diff successfully check diff failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:02 CST 2024] <<<<<< test case sequence_safe_mode success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } check diff failed 6-th time, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 2-th time, retry later check diff failed 7-th time, retry later dmctl test cmd: "stop-task test" [Thu May 30 15:47:05 CST 2024] <<<<<< finish DM-015 >>>>>> check diff successfully check diff successfully check diff successfully 1 dm-master alive 0 dm-worker alive 2 dm-syncer alive [Thu May 30 15:47:06 CST 2024] <<<<<< start DM-016 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check diff failed 3-th time, retry later wait process dm-master.test exit... check diff failed 8-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:07 CST 2024] <<<<<< test case dm_syncer success! >>>>>> start running case: [downstream_diff_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed at last dmctl test cmd: "query-status test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check diff failed 9-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "stop-task test" [Thu May 30 15:47:10 CST 2024] <<<<<< finish DM-016 >>>>>> [Thu May 30 15:47:11 CST 2024] <<<<<< start DM-017 >>>>>> use sync_diff_inspector to check increment data check diff successfully check diff successfully [Thu May 30 15:46:59 CST 2024] <<<<<< finish test_pause_task >>>>>> [Thu May 30 15:46:59 CST 2024] <<<<<< start test_multi_task_reduce_and_restart_worker >>>>>> [Thu May 30 15:46:59 CST 2024] <<<<<< start test_multi_task_running >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master [Thu May 30 15:47:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>> [Thu May 30 15:47:10 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>> [Thu May 30 15:47:10 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 10-th time, retry later wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source1.yaml" [Thu May 30 15:47:12 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-worker2.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive check diff failed at last dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Thu May 30 15:47:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:47:13 CST 2024] <<<<<< finish DM-119 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:13 CST 2024] <<<<<< finish test_expression_filter >>>>>> [Thu May 30 15:47:13 CST 2024] <<<<<< start test_json_expression >>>>>> [Thu May 30 15:47:13 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/source2.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml" dmctl test cmd: "stop-task test" [Thu May 30 15:47:14 CST 2024] <<<<<< start DM-120 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:47:14 CST 2024] <<<<<< finish DM-017 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_diff_index/conf/dm-task.yaml --remove-meta" [Thu May 30 15:47:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:47:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:47:15 CST 2024] <<<<<< start DM-018 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" got=2 expected=2 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml" check diff failed 1-th time, retry later run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later [Thu May 30 15:47:17 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task-expression-filter.yaml --remove-meta" rpc addr 127.0.0.1:8264 is alive [Thu May 30 15:47:18 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time [Thu May 30 15:47:18 CST 2024] <<<<<< finish DM-120 pessimistic >>>>>> [Thu May 30 15:47:18 CST 2024] <<<<<< start DM-120 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Thu May 30 15:47:19 CST 2024] <<<<<< finish DM-018 >>>>>> check diff failed 1-th time, retry later rpc addr 127.0.0.1:18262 is alive [Thu May 30 15:47:19 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit check diff failed 1-th time, retry later [Thu May 30 15:47:20 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml " check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:47:22 CST 2024] <<<<<< finish DM-019 >>>>>> [Thu May 30 15:47:22 CST 2024] <<<<<< start DM-019 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully wait process dm-worker.test exit... check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:47:22 CST 2024] <<<<<< finish DM-120 optimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:23 CST 2024] <<<<<< test case downstream_diff_index success! >>>>>> start running case: [downstream_more_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit [Thu May 30 15:47:23 CST 2024] <<<<<< start DM-121 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" [Thu May 30 15:47:23 CST 2024] <<<<<< finish DM-019 >>>>>> wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:24 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:47:24 CST 2024] <<<<<< start DM-020 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/downstream_more_column/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/conf/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:26 CST 2024] <<<<<< finish test_json_expression >>>>>> [Thu May 30 15:47:26 CST 2024] <<<<<< start test_fail_job_between_event >>>>>> [Thu May 30 15:47:26 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/downstream_more_column/dm-task-incremental.yaml --remove-meta" check diff failed 2-th time, retry later use sync_diff_inspector to check increment data check diff successfully rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully [Thu May 30 15:47:28 CST 2024] <<<<<< finish test_multi_task_running >>>>>> start dumping SQLs into source "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) restart unuse worker3 try to kill worker port 8264 dmctl test cmd: "stop-task test" [Thu May 30 15:47:28 CST 2024] <<<<<< finish DM-020 >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d downstream_more_column1 -t t1 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/downstream_more_column/data/schema.sql" dmctl test cmd: "resume-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" wait process dm-worker3 exit... dmctl test cmd: "query-status test" got=2 expected=2 [Thu May 30 15:47:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive [Thu May 30 15:47:29 CST 2024] <<<<<< start DM-021 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check diff failed 3-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" wait process dm-worker3 exit... process dm-worker3 already exit dmctl test cmd: "list-member --worker --name=worker3" got=1 expected=1 start dm-worker3 [Thu May 30 15:47:30 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8264 is alive restart unuse worker4 try to kill worker port 18262 wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-task.yaml --remove-meta" check diff failed 4-th time, retry later wait process dm-worker.test exit... wait process dm-worker4 exit... check log contain failed 1-th time, retry later dmctl test cmd: "stop-task test" [Thu May 30 15:47:33 CST 2024] <<<<<< finish DM-021 >>>>>> wait process dm-worker.test exit... wait process dm-worker4 exit... process dm-worker4 already exit dmctl test cmd: "list-member --worker --name=worker4" got=1 expected=1 start dm-worker4 [Thu May 30 15:47:33 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time check diff failed 5-th time, retry later [Thu May 30 15:47:34 CST 2024] <<<<<< start DM-022 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-no-routes.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:34 CST 2024] <<<<<< test case downstream_more_column success! >>>>>> start running case: [drop_column_with_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" rpc addr 127.0.0.1:18262 is alive restart unuse worker5 try to kill worker port 18263 got=3 expected=3 check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... wait process dm-worker5 exit... check diff failed 6-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker5 exit... process dm-worker5 already exit dmctl test cmd: "list-member --worker --name=worker5" got=1 expected=1 start dm-worker5 [Thu May 30 15:47:37 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" check diff failed 7-th time, retry later rpc addr 127.0.0.1:18263 is alive try to kill worker port 8262 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/drop_column_with_index/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-task.yaml --remove-meta" [Thu May 30 15:47:38 CST 2024] <<<<<< finish DM-022 >>>>>> wait process dm-worker.test exit... [Thu May 30 15:47:39 CST 2024] <<<<<< start DM-023 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker1 exit... check diff failed 8-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit worker1 was killed dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-worker.test exit... got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 got=1 expected=1 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:41 CST 2024] <<<<<< finish test_fail_job_between_event >>>>>> [Thu May 30 15:47:41 CST 2024] <<<<<< start test_session_config >>>>>> [Thu May 30 15:47:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> [Thu May 30 15:47:41 CST 2024] <<<<<< finish DM-023 >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 9-th time, retry later [Thu May 30 15:47:42 CST 2024] <<<<<< start DM-026 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:43 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:47:44 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff failed 10-th time, retry later [Thu May 30 15:47:44 CST 2024] <<<<<< finish DM-026 >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" [Thu May 30 15:47:45 CST 2024] <<<<<< start DM-027 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" check diff failed at last dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again [Thu May 30 15:47:46 CST 2024] <<<<<< finish DM-121 pessimistic >>>>>> [Thu May 30 15:47:46 CST 2024] <<<<<< start DM-121 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:47:47 CST 2024] <<<<<< finish DM-027 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check diff successfully dmctl test cmd: "query-status t-ร‹!s`t" [Thu May 30 15:47:48 CST 2024] <<<<<< start DM-028 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta" got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:47:50 CST 2024] <<<<<< finish DM-028 >>>>>> wait process dm-master.test exit... check diff failed 2-th time, retry later wait process dm-master.test exit... [Thu May 30 15:47:51 CST 2024] <<<<<< start DM-030 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:54 CST 2024] <<<<<< test case drop_column_with_index success! >>>>>> start running case: [duplicate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:54 CST 2024] <<<<<< finish test_session_config >>>>>> [Thu May 30 15:47:54 CST 2024] <<<<<< start test_query_timeout >>>>>> 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:47:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 4-th time, retry later [Thu May 30 15:47:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:47:55 CST 2024] <<<<<< finish DM-030 pessimistic >>>>>> [Thu May 30 15:47:55 CST 2024] <<<<<< start DM-030 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:47:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" check diff failed 5-th time, retry later [Thu May 30 15:47:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:47:57 CST 2024] <<<<<< finish DM-030 optimistic >>>>>> [Thu May 30 15:47:58 CST 2024] <<<<<< start DM-031 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" check diff failed 6-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Thu May 30 15:47:59 CST 2024] <<<<<< finish DM-031 pessimistic >>>>>> [Thu May 30 15:47:59 CST 2024] <<<<<< start DM-031 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 2-th time, retry later check diff failed 7-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:48:01 CST 2024] <<<<<< finish DM-031 optimistic >>>>>> check log contain failed 3-th time, retry later [Thu May 30 15:48:02 CST 2024] <<<<<< start DM-032 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" check diff failed 8-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:04 CST 2024] <<<<<< finish DM-032 optimistic >>>>>> check log contain failed 4-th time, retry later check diff failed 9-th time, retry later [Thu May 30 15:48:05 CST 2024] <<<<<< start DM-033 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 5-th time, retry later check diff failed 10-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 6-th time, retry later check diff failed at last dmctl test cmd: "stop-task test" [Thu May 30 15:48:09 CST 2024] <<<<<< finish DM-121 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:09 CST 2024] <<<<<< finish DM-033 pessimistic >>>>>> [Thu May 30 15:48:10 CST 2024] <<<<<< start DM-122 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" [Thu May 30 15:48:10 CST 2024] <<<<<< start DM-034 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 7-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:48:12 CST 2024] <<<<<< finish DM-034 pessimistic >>>>>> [Thu May 30 15:48:12 CST 2024] <<<<<< start DM-034 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 8-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Thu May 30 15:48:14 CST 2024] <<<<<< finish DM-034 optimistic >>>>>> check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:14 CST 2024] <<<<<< finish DM-122 pessimistic >>>>>> [Thu May 30 15:48:14 CST 2024] <<<<<< start DM-122 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 9-th time, retry later [Thu May 30 15:48:15 CST 2024] <<<<<< start DM-035 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:16 CST 2024] <<<<<< finish DM-122 optimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later [Thu May 30 15:48:17 CST 2024] <<<<<< start DM-123 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 10-th time, retry later dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Thu May 30 15:48:19 CST 2024] <<<<<< finish DM-035 optimistic >>>>>> check log contain failed 11-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... check log contain failed 12-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:21 CST 2024] <<<<<< finish DM-123 pessimistic >>>>>> [Thu May 30 15:48:21 CST 2024] <<<<<< start DM-123 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later check log contain failed 13-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:48:25 CST 2024] <<<<<< test case shardddl1_1 success! >>>>>> start running case: [shardddl2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:48:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> check log contain failed 14-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:26 CST 2024] <<<<<< finish DM-123 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Thu May 30 15:48:27 CST 2024] <<<<<< start DM-124 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 15-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:48:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source1.yaml" [Thu May 30 15:48:29 CST 2024] <<<<<< finish DM-124 optimistic >>>>>> [Thu May 30 15:48:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 16-th time, retry later [Thu May 30 15:48:30 CST 2024] <<<<<< start DM-125 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl2/source2.yaml" [Thu May 30 15:48:31 CST 2024] <<<<<< start DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 17-th time, retry later check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker 1 check log contain failed 18-th time, retry later wait process dm-worker1 exit... check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Thu May 30 15:48:34 CST 2024] <<<<<< finish DM-125 optimistic >>>>>> wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:48:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 19-th time, retry later [Thu May 30 15:48:35 CST 2024] <<<<<< start DM-126 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "query-status t-ร‹!s`t" check log contain failed 20-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 restart dm-master got=2 expected=2 check diff successfully dmctl test cmd: "stop-task t-ร‹!s`t" wait process dm-master exit... check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Thu May 30 15:48:39 CST 2024] <<<<<< finish DM-126 optimistic >>>>>> check log contain failed 21-th time, retry later wait process dm-master exit... [Thu May 30 15:48:40 CST 2024] <<<<<< start DM-127 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master exit... process dm-master already exit check log contain failed 22-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later [Thu May 30 15:48:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 23-th time, retry later check diff successfully dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "stop-task test" [Thu May 30 15:48:45 CST 2024] <<<<<< finish DM-127 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 24-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:48:46 CST 2024] <<<<<< start DM-128 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 0-th time, will retry again 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit check log contain failed 25-th time, retry later got=1 expected=1 got=1 expected=1 restart dm-worker 1 wait process dm-worker.test exit... wait process dm-worker1 exit... check log contain failed 26-th time, retry later wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:50 CST 2024] <<<<<< finish DM-128 pessimistic >>>>>> [Thu May 30 15:48:50 CST 2024] <<<<<< start DM-128 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:48:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:48:51 CST 2024] <<<<<< finish test_query_timeout >>>>>> [Thu May 30 15:48:51 CST 2024] <<<<<< start test_regexpr_router regexpr-task.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:48:51 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 27-th time, retry later got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later [Thu May 30 15:48:52 CST 2024] <<<<<< finish DM-128 optimistic >>>>>> rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:48:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:48:53 CST 2024] <<<<<< start DM-129 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:48:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 28-th time, retry later check diff failed 2-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" check log contain failed 29-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta" [Thu May 30 15:48:56 CST 2024] <<<<<< finish DM-DROP_COLUMN_EXEC_ERROR optimistic >>>>>> [Thu May 30 15:48:56 CST 2024] <<<<<< start DM-INIT_SCHEMA optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:57 CST 2024] <<<<<< finish DM-129 pessimistic >>>>>> [Thu May 30 15:48:57 CST 2024] <<<<<< start DM-129 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later check log contain failed 1-th time, retry later wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:48:59 CST 2024] <<<<<< finish DM-129 optimistic >>>>>> wait process dm-master.test exit... check diff successfully [Thu May 30 15:49:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time restart dm-master [Thu May 30 15:49:00 CST 2024] <<<<<< start DM-130 pessimistic >>>>>> wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" wait process dm-master exit... wait process dm-worker.test exit... use sync_diff_inspector to check increment data check diff successfully check diff successfully data checked after one worker was killed try to kill worker port 8263 wait process dm-worker2 exit... wait process dm-worker2 exit... process dm-worker2 already exit worker2 was killed dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 [Thu May 30 15:48:59 CST 2024] <<<<<< finish test_multi_task_reduce_and_restart_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master exit... process dm-master already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" dmctl test cmd: "stop-task test" [Thu May 30 15:49:02 CST 2024] <<<<<< finish DM-130 pessimistic >>>>>> [Thu May 30 15:49:02 CST 2024] <<<<<< start DM-130 optimistic >>>>>> wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:49:03 CST 2024] <<<<<< finish test_regexpr_router regexpr-task.yaml >>>>>> [Thu May 30 15:49:03 CST 2024] <<<<<< start test_regexpr_router regexpr-task-lightning.yaml >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:49:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:49:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> [Thu May 30 15:49:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 check diff failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:49:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb1 --action skip" dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb` -s mysql-replica-02 -d shardddl1 -t tb2 --action skip" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" check diff successfully dmctl test cmd: "stop-task test" check diff failed 2-th time, retry later [Thu May 30 15:49:07 CST 2024] <<<<<< finish DM-130 optimistic >>>>>> wait process dm-master.test exit... [Thu May 30 15:49:07 CST 2024] <<<<<< finish DM-INIT_SCHEMA optimistic >>>>>> [Thu May 30 15:49:07 CST 2024] <<<<<< start DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:49:08 CST 2024] <<<<<< test case ha_cases2 success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker 2 [Pipeline] } [Pipeline] // container [Pipeline] } check diff failed 3-th time, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process dm-master.test exit... wait process dm-worker2 exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... check diff failed 4-th time, retry later wait process dm-worker2 exit... process dm-worker2 already exit [Thu May 30 15:49:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive check log contain failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:49:13 CST 2024] <<<<<< test case shardddl4 success! >>>>>> start running case: [shardddl4_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/run.sh... Verbose mode = false check diff failed 5-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "query-status test" [Thu May 30 15:49:14 CST 2024] <<<<<< finish test_regexpr_router regexpr-task-lightning.yaml >>>>>> [Thu May 30 15:49:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>> got=1 expected=1 restart dm-master wait for rpc addr 127.0.0.1:8261 alive the 1-th time 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:49:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 6-th time, retry later wait process dm-master exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:49:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:49:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" [Thu May 30 15:49:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source1.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml" [Thu May 30 15:49:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Thu May 30 15:49:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4_1/source2.yaml" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task /tmp/dm_test/all_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 restart dm-worker 2 wait process dm-worker.test exit... [Thu May 30 15:49:21 CST 2024] <<<<<< start DM-TABLE_CHECKPOINT_BACKWARD optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit check diff successfully [Thu May 30 15:49:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker2 exit... rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:49:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait process dm-worker2 exit... process dm-worker2 already exit [Thu May 30 15:49:23 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "pause-task t-ร‹!s`t" dmctl test cmd: "operate-schema set -s mysql-replica-01 t-ร‹!s`t -d all_mode -t no_diff /tmp/dm_test/all_mode/schema.sql" dmctl test cmd: "resume-task t-ร‹!s`t" wait pattern dm-worker1.toml exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" wait pattern dm-worker1.toml exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" got=3 expected=3 check diff failed 1-th time, retry later wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... wait pattern dm-worker1.toml exit... pattern dm-worker1.toml already exit [Thu May 30 15:49:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:49:27 CST 2024] <<<<<< finish DM-DROP_COLUMN_ALL_DONE optimistic >>>>>> [Thu May 30 15:49:27 CST 2024] <<<<<< start DM-RECOVER_LOCK optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... wait pattern dm-worker2.toml exit... check log contain failed 2-th time, retry later wait pattern dm-worker2.toml exit... pattern dm-worker2.toml already exit [Thu May 30 15:49:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive check log contain failed 3-th time, retry later check log contain failed 1-th time, retry later check log contain failed 4-th time, retry later restart dm-master wait process dm-master exit... check log contain failed 5-th time, retry later wait process dm-master exit... process dm-master already exit check log contain failed 6-th time, retry later [Thu May 30 15:49:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 7-th time, retry later rpc addr 127.0.0.1:8261 is alive restart dm-master after restart dm-worker, task should resume automatically dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml" check log contain failed 8-th time, retry later wait process dm-master exit... wait process dm-master exit... process dm-master already exit HTTP 127.0.0.1:8261/apis/v1alpha1/status/t-ร‹!s`t is alive check log contain failed 9-th time, retry later [Thu May 30 15:49:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 10-th time, retry later rpc addr 127.0.0.1:8261 is alive check diff failed 1-th time, retry later wait process tidb-server exit... process tidb-server already exit check log contain failed 11-th time, retry later check diff successfully dmctl test cmd: "shard-ddl-lock" Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:49:49 CST 2024] <<<<<< finish DM-RECOVER_LOCK optimistic >>>>>> run DM_DropAddColumn case #0 [Thu May 30 15:49:49 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 12-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 180 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%' check diff successfully check log contain failed 1-th time, retry later check log contain failed 13-th time, retry later check diff failed 1-th time, retry later dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check log contain failed 14-th time, retry later check diff successfully dmctl test cmd: "pause-relay -s mysql-replica-01" dmctl test cmd: "resume-relay -s mysql-replica-01" check log contain failed 15-th time, retry later check diff failed 2-th time, retry later check log contain failed 16-th time, retry later relay logs dm-it-f736575a-2dc6-454c-841b-c486aca56cd8-rjd34-fwhg0-bin.000002 relay.meta check diff successfully check dump files have been cleaned ls: cannot access /tmp/dm_test/all_mode/worker2/dumped_data.t-ร‹!s`t: No such file or directory worker2 auto removed dump files check no password in log dmctl test cmd: "query-status t-ร‹!s`t" got=1 expected=1 dmctl test cmd: "stop-task t-ร‹!s`t" check diff failed 3-th time, retry later matched matched [Thu May 30 15:49:57 CST 2024] <<<<<< start test_source_and_target_with_empty_gtid >>>>>> 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check log contain failed 17-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:50:00 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #1 [Thu May 30 15:50:00 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit check log contain failed 18-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully restart dm-master wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:50:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master exit... check log contain failed 19-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:50:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/all_mode/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master exit... process dm-master already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml" got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 gtid is empty start task and check stage dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task-no-gtid.yaml --remove-meta=true" check log contain failed 20-th time, retry later [Thu May 30 15:50:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data check diff successfully ERROR 1146 (42S02) at line 1: Table 'all_mode.t2' doesn't exist run tidb sql failed 1-th time, retry later check log contain failed 21-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check log contain failed 1-th time, retry later check log contain failed 22-th time, retry later dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check log contain failed 23-th time, retry later check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later check log contain failed 24-th time, retry later check diff successfully dmctl test cmd: "stop-task test" check diff failed 2-th time, retry later [Thu May 30 15:50:14 CST 2024] <<<<<< finish DM-TABLE_CHECKPOINT_BACKWARD optimistic >>>>>> [Thu May 30 15:50:14 CST 2024] <<<<<< start DM-RESYNC_NOT_FLUSHED optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 25-th time, retry later check diff failed 3-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker1 wait process worker1 exit... check log contain failed 26-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" wait process worker1 exit... process worker1 already exit [Thu May 30 15:50:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> dmctl test cmd: "resume-task test" wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:50:18 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #2 [Thu May 30 15:50:18 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive restart dm-worker2 check log contain failed 27-th time, retry later dmctl test cmd: "query-status test" wait process worker2 exit... got=2 expected=2 check diff successfully restart dm-master wait process dm-master exit... wait process worker2 exit... process worker2 already exit [Thu May 30 15:50:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check log contain failed 28-th time, retry later rpc addr 127.0.0.1:8263 is alive wait process dm-master exit... process dm-master already exit check log contain failed 29-th time, retry later [Thu May 30 15:50:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 30-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later check log contain failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully [Thu May 30 15:50:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" check diff failed 3-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:50:34 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #3 [Thu May 30 15:50:34 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" [Thu May 30 15:50:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:50:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 check diff successfully binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master wait process dm-worker.test exit... wait process dm-master exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:50:39 CST 2024] <<<<<< test case duplicate_event success! >>>>>> start running case: [expression_filter] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:50:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:50:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml" wait process dm-master exit... wait process dm-master exit... process dm-master already exit dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task2.yaml " [Thu May 30 15:50:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=2 expected=2 got=1 expected=1 dmctl test cmd: "stop-task test" ls: cannot access /tmp/dm_test/expression_filter/worker1/schema-tracker*: No such file or directory schema tracker path has been cleaned 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit check diff failed 2-th time, retry later wait process dm-worker.test exit... wait process dm-worker.test exit... check diff failed 3-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:50:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:50:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/source1.yaml" [Thu May 30 15:50:55 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #4 [Thu May 30 15:50:55 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/expression_filter/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later check diff successfully restart dm-worker1 wait process worker1 exit... wait process worker1 exit... process worker1 already exit [Thu May 30 15:50:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "shard-ddl-lock" got=1 expected=1 restart dm-master rpc addr 127.0.0.1:8262 is alive restart dm-worker2 wait process dm-master exit... wait process worker2 exit... wait process dm-master exit... process dm-master already exit wait process worker2 exit... process worker2 already exit [Thu May 30 15:51:01 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-task test" [Thu May 30 15:51:03 CST 2024] <<<<<< finish DM-RESYNC_NOT_FLUSHED optimistic >>>>>> [Thu May 30 15:51:03 CST 2024] <<<<<< start DM-RESYNC_TXN_INTERRUPT optimistic >>>>>> wait process dm-master.test exit... [Thu May 30 15:51:03 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 restart dm-worker1 wait process dm-worker.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process worker1 exit... wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 check diff failed 1-th time, retry later wait process worker1 exit... process worker1 already exit [Thu May 30 15:51:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:51:06 CST 2024] <<<<<< test case expression_filter success! >>>>>> start running case: [extend_column] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: sql [Thu May 30 15:51:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive restart dm-worker2 rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:51:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... check diff failed 2-th time, retry later rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:51:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process worker2 exit... process worker2 already exit [Thu May 30 15:51:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" rpc addr 127.0.0.1:8263 is alive begin; insert into shardddl1.tb2 values (1,1); insert into shardddl1.tb2 values (2,2); insert into shardddl1.tb2 values (3,3); insert into shardddl1.tb2 values (4,4); insert into shardddl1.tb2 values (5,5); insert into shardddl1.tb2 values (6,6); insert into shardddl1.tb2 values (7,7); insert into shardddl1.tb2 values (8,8); insert into shardddl1.tb2 values (9,9); insert into shardddl1.tb2 values (10,10); commit; begin; insert into shardddl1.t_1 values (11,11); insert into shardddl1.t_1 values (12,12); insert into shardddl1.t_1 values (13,13); insert into shardddl1.t_1 values (14,14); insert into shardddl1.t_1 values (15,15); insert into shardddl1.t_1 values (16,16); insert into shardddl1.t_1 values (17,17); insert into shardddl1.t_1 values (18,18); insert into shardddl1.t_1 values (19,19); insert into shardddl1.t_1 values (20,20); insert into shardddl1.t_1 values (21,21); insert into shardddl1.t_1 values (22,22); insert into shardddl1.t_1 values (23,23); insert into shardddl1.t_1 values (24,24); insert into shardddl1.t_1 values (25,25); insert into shardddl1.t_1 values (26,26); insert into shardddl1.t_1 values (27,27); insert into shardddl1.t_1 values (28,28); insert into shardddl1.t_1 values (29,29); insert into shardddl1.t_1 values (30,30); insert into shardddl1.t_1 values (31,31); insert into shardddl1.t_1 values (32,32); insert into shardddl1.t_1 values (33,33); insert into shardddl1.t_1 values (34,34); insert into shardddl1.t_1 values (35,35); insert into shardddl1.t_1 values (36,36); insert into shardddl1.t_1 values (37,37); insert into shardddl1.t_1 values (38,38); insert into shardddl1.t_1 values (39,39); insert into shardddl1.t_1 values (40,40); insert into shardddl1.t_1 values (41,41); insert into shardddl1.t_1 values (42,42); insert into shardddl1.t_1 values (43,43); insert into shardddl1.t_1 values (44,44); insert into shardddl1.t_1 values (45,45); insert into shardddl1.t_1 values (46,46); insert into shardddl1.t_1 values (47,47); insert into shardddl1.t_1 values (48,48); insert into shardddl1.t_1 values (49,49); insert into shardddl1.t_1 values (50,50); commit; begin; insert into shardddl1.tb1 values (51,51); insert into shardddl1.tb1 values (52,52); insert into shardddl1.tb1 values (53,53); insert into shardddl1.tb1 values (54,54); insert into shardddl1.tb1 values (55,55); insert into shardddl1.tb1 values (56,56); insert into shardddl1.tb1 values (57,57); insert into shardddl1.tb1 values (58,58); insert into shardddl1.tb1 values (59,59); insert into shardddl1.tb1 values (60,60); commit; begin; insert into shardddl1.t_1 values (61,61); insert into shardddl1.t_1 values (62,62); insert into shardddl1.t_1 values (63,63); insert into shardddl1.t_1 values (64,64); insert into shardddl1.t_1 values (65,65); insert into shardddl1.t_1 values (66,66); insert into shardddl1.t_1 values (67,67); insert into shardddl1.t_1 values (68,68); insert into shardddl1.t_1 values (69,69); insert into shardddl1.t_1 values (70,70); commit; check diff failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" check diff failed 3-th time, retry later check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later check diff failed at last dmctl test cmd: "binlog skip test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:51:15 CST 2024] <<<<<< finish DM-DropAddColumn optimistic >>>>>> run DM_DropAddColumn case #5 [Thu May 30 15:51:15 CST 2024] <<<<<< start DM-DropAddColumn optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/double-source-optimistic.yaml --remove-meta" check diff failed 3-th time, retry later 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully check log contain failed 1-th time, retry later wait process dm-master.test exit... check diff successfully restart dm-worker1 wait process dm-master.test exit... process dm-master.test already exit wait process worker1 exit... dmctl test cmd: "shard-ddl-lock" wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 restart dm-master wait process worker1 exit... process worker1 already exit [Thu May 30 15:51:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... wait process dm-master exit... rpc addr 127.0.0.1:8262 is alive restart dm-worker2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit running extend_column case with import_mode: loader [Thu May 30 15:51:20 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master exit... process dm-master already exit wait process worker2 exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process worker2 exit... process worker2 already exit [Thu May 30 15:51:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:51:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:51:23 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive begin; insert into shardddl1.tb2 values (101,101); insert into shardddl1.tb2 values (102,102); insert into shardddl1.tb2 values (103,103); insert into shardddl1.tb2 values (104,104); insert into shardddl1.tb2 values (105,105); insert into shardddl1.tb2 values (106,106); insert into shardddl1.tb2 values (107,107); insert into shardddl1.tb2 values (108,108); insert into shardddl1.tb2 values (109,109); insert into shardddl1.tb2 values (110,110); commit; begin; insert into shardddl1.tb1 values (111,111); insert into shardddl1.tb1 values (112,112); insert into shardddl1.tb1 values (113,113); insert into shardddl1.tb1 values (114,114); insert into shardddl1.tb1 values (115,115); insert into shardddl1.tb1 values (116,116); insert into shardddl1.tb1 values (117,117); insert into shardddl1.tb1 values (118,118); insert into shardddl1.tb1 values (119,119); insert into shardddl1.tb1 values (120,120); commit; begin; insert into shardddl1.tb2 values (121,121); insert into shardddl1.tb2 values (122,122); insert into shardddl1.tb2 values (123,123); insert into shardddl1.tb2 values (124,124); insert into shardddl1.tb2 values (125,125); insert into shardddl1.tb2 values (126,126); insert into shardddl1.tb2 values (127,127); insert into shardddl1.tb2 values (128,128); insert into shardddl1.tb2 values (129,129); insert into shardddl1.tb2 values (130,130); commit; begin; insert into shardddl1.t_1 values (131,131); insert into shardddl1.t_1 values (132,132); insert into shardddl1.t_1 values (133,133); insert into shardddl1.t_1 values (134,134); insert into shardddl1.t_1 values (135,135); insert into shardddl1.t_1 values (136,136); insert into shardddl1.t_1 values (137,137); insert into shardddl1.t_1 values (138,138); insert into shardddl1.t_1 values (139,139); insert into shardddl1.t_1 values (140,140); commit; check diff successfully rpc addr 127.0.0.1:8262 is alive [Thu May 30 15:51:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/extend_column/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time begin; insert into shardddl1.tb2 values (201,201); insert into shardddl1.tb2 values (202,202); insert into shardddl1.tb2 values (203,203); insert into shardddl1.tb2 values (204,204); insert into shardddl1.tb2 values (205,205); insert into shardddl1.tb2 values (206,206); insert into shardddl1.tb2 values (207,207); insert into shardddl1.tb2 values (208,208); insert into shardddl1.tb2 values (209,209); insert into shardddl1.tb2 values (210,210); commit; begin; insert into shardddl1.tb1 values (211,211); insert into shardddl1.tb1 values (212,212); insert into shardddl1.tb1 values (213,213); insert into shardddl1.tb1 values (214,214); insert into shardddl1.tb1 values (215,215); insert into shardddl1.tb1 values (216,216); insert into shardddl1.tb1 values (217,217); insert into shardddl1.tb1 values (218,218); insert into shardddl1.tb1 values (219,219); insert into shardddl1.tb1 values (220,220); commit; begin; insert into shardddl1.tb2 values (221,221); insert into shardddl1.tb2 values (222,222); insert into shardddl1.tb2 values (223,223); insert into shardddl1.tb2 values (224,224); insert into shardddl1.tb2 values (225,225); insert into shardddl1.tb2 values (226,226); insert into shardddl1.tb2 values (227,227); insert into shardddl1.tb2 values (228,228); insert into shardddl1.tb2 values (229,229); insert into shardddl1.tb2 values (230,230); commit; begin; insert into shardddl1.t_1 values (231,231); insert into shardddl1.t_1 values (232,232); insert into shardddl1.t_1 values (233,233); insert into shardddl1.t_1 values (234,234); insert into shardddl1.t_1 values (235,235); insert into shardddl1.t_1 values (236,236); insert into shardddl1.t_1 values (237,237); insert into shardddl1.t_1 values (238,238); insert into shardddl1.t_1 values (239,239); insert into shardddl1.t_1 values (240,240); commit; check diff failed 1-th time, retry later rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/extend_column/source2.yaml" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 0-th time, will retry again check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /tmp/dm_test/extend_column/dm-task.yaml --remove-meta" [Thu May 30 15:51:26 CST 2024] <<<<<< finish DM-RESYNC_TXN_INTERRUPT optimistic >>>>>> [Thu May 30 15:51:26 CST 2024] <<<<<< start DM-STRICT_OPTIMISTIC_SINGLE_SOURCE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/single-source-strict-optimistic.yaml --remove-meta" run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 1-th time, will retry again [Thu May 30 15:51:28 CST 2024] <<<<<< finish DM-STRICT_OPTIMISTIC_SINGLE_SOURCE optimistic >>>>>> [Thu May 30 15:51:28 CST 2024] <<<<<< start DM-STRICT_OPTIMISTIC_DOUBLE_SOURCE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-strict-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 2-th time, will retry again 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 dmctl test cmd: "stop-task test" [Thu May 30 15:51:31 CST 2024] <<<<<< finish DM-STRICT_OPTIMISTIC_DOUBLE_SOURCE optimistic >>>>>> [Thu May 30 15:51:31 CST 2024] <<<<<< start DM-131 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 3-th time, will retry again check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:51:33 CST 2024] <<<<<< finish DM-131 optimistic >>>>>> wait process dm-worker.test exit... [Thu May 30 15:51:34 CST 2024] <<<<<< start DM-132 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 4-th time, will retry again dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:51:35 CST 2024] <<<<<< test case extend_column success! >>>>>> start running case: [fake_rotate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:51:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:51:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 5-th time, will retry again rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/fake_rotate_event/source1.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-task.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:51:38 CST 2024] <<<<<< finish DM-132 pessimistic >>>>>> [Thu May 30 15:51:38 CST 2024] <<<<<< start DM-132 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 6-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 check diff successfully kill dm-worker dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" [Thu May 30 15:51:40 CST 2024] <<<<<< finish DM-132 optimistic >>>>>> got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 7-th time, will retry again wait process dm-worker1 exit... process dm-worker1 already exit [Thu May 30 15:51:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/fake_rotate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Thu May 30 15:51:41 CST 2024] <<<<<< start DM-133 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 8-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=1 expected=1 <<<<<< test_source_and_target_with_empty_gtid success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check diff successfully dmctl test cmd: "stop-task test" got=0 expected=1 command: query-status test because schema conflict detected count: 0 != expected: 1, failed the 9-th time, will retry again wait process dm-worker.test exit... [Thu May 30 15:51:46 CST 2024] <<<<<< finish DM-133 pessimistic >>>>>> [Thu May 30 15:51:46 CST 2024] <<<<<< start DM-133 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:51:46 CST 2024] <<<<<< test case all_mode success! >>>>>> [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-545/tiflow-dm already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } wait process dm-worker.test exit... [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-01", "worker": "worker1", "result": null, "relayStatus": null }, "subTaskStatus": [ { "name": "test", "stage": "Paused", "unit": "Sync", "result": { "isCanceled": false, "errors": [ { "ErrCode": 42501, "ErrClass": "ha", "ErrScope": "internal", "ErrLevel": "high", "Message": "startLocation: [position: (dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin.000001, 42228), gtid-set: bc4e5ee7-1e57-11ef-83ce-667f93470dda:1-194], endLocation: [position: (dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin.000001, 42353), gtid-set: bc4e5ee7-1e57-11ef-83ce-667f93470dda:1-195], origin SQL: [alter table shardddl1.tb1 add column b int after a]: fail to do etcd txn operation: txn commit failed", "RawCause": "rpc error: code = Unavailable desc = error reading from server: EOF", "Workaround": "Please check dm-master's node status and the network between this node and dm-master" } ], "detail": null }, "unresolvedDDLLockID": "", "sync": { "totalEvents": "12", "totalTps": "0", "recentTps": "0", "masterBinlog": "(dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin.000001, 42353)", "masterBinlogGtid": "bc4e5ee7-1e57-11ef-83ce-667f93470dda:1-195", "syncerBinlog": "(dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin.000001, 42163)", "syncerBinlogGtid": "bc4e5ee7-1e57-11ef-83ce-667f93470dda:1-194", "blockingDDLs": [ ], "unresolvedGroups": [ ], "synced": false, "binlogType": "remote", "secondsBehindMaster": "0", "blockDDLOwner": "", "conflictMsg": "", "totalRows": "12", "totalRps": "0", "recentRps": "0" }, "validation": null } ] }, { "result": true, "msg": "", "sourceStatus": { "source": "mysql-replica-02", "worker": "worker2", "result": null, "relayStatus": { "masterBinlog": "(dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin.000001, 39206)", "masterBinlogGtid": "bed69e82-1e57-11ef-8248-667f93470dda:1-167", "relaySubDir": "bed69e82-1e57-11ef-8248-667f93470dda.000001", "relayBinlog": "(dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin.000001, 39206)", "relayBinlogGtid": "bed69e82-1e57-11ef-8248-667f93470dda:1-167", "relayCatchUpMaster": true, "stage": "Running", "result": null } }, "subTaskStatus": [ { "name": "test", "stage": "Running", "unit": "Sync", "result": null, "unresolvedDDLLockID": "", "sync": { "totalEvents": "6", "totalTps": "0", "recentTps": "0", "masterBinlog": "(dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin.000001, 39206)", "masterBinlogGtid": "bed69e82-1e57-11ef-8248-667f93470dda:1-167", "syncerBinlog": "(dm-it-33c68001-94a4-4ffc-b622-0669e37e0319-3vxhq-w8gzs-bin|000001.000001, 38926)", "syncerBinlogGtid": "bed69e82-1e57-11ef-8248-667f93470dda:1-166", "blockingDDLs": [ ], "unresolvedGroups": [ ], "synced": false, "binlogType": "local", "secondsBehindMaster": "0", "blockDDLOwner": "", "conflictMsg": "", "totalRows": "6", "totalRps": "0", "recentRps": "0" }, "validation": null } ] } ] } PASS coverage: 4.0% of statements in github.com/pingcap/tiflow/dm/... curl: (7) Failed connect to 127.0.0.1:8361; Connection refused curl: (7) Failed connect to 127.0.0.1:8461; Connection refused curl: (7) Failed connect to 127.0.0.1:8561; Connection refused curl: (7) Failed connect to 127.0.0.1:8661; Connection refused curl: (7) Failed connect to 127.0.0.1:8761; Connection refused curl: (7) Failed connect to 127.0.0.1:8264; Connection refused curl: (7) Failed connect to 127.0.0.1:18262; Connection refused curl: (7) Failed connect to 127.0.0.1:18263; Connection refused make: *** [dm_integration_test_in_group] Error 1 dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // withEnv [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // stage [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // dir check diff successfully dmctl test cmd: "stop-task test" Post stage [Pipeline] sh wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:51:47 CST 2024] <<<<<< test case fake_rotate_event success! >>>>>> start running case: [foreign_key] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Thu May 30 15:51:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Thu May 30 15:51:48 CST 2024] <<<<<< finish DM-133 optimistic >>>>>> + ls /tmp/dm_test cov.shardddl1.dmctl.1717054894.644.out cov.shardddl1.dmctl.1717054896.760.out cov.shardddl1.dmctl.1717054900.1041.out cov.shardddl1.dmctl.1717054901.1082.out cov.shardddl1.dmctl.1717054926.4268.out cov.shardddl1.dmctl.1717054929.4596.out cov.shardddl1.dmctl.1717054931.4636.out cov.shardddl1.dmctl.1717054962.5136.out cov.shardddl1.dmctl.1717054965.5456.out cov.shardddl1.dmctl.1717054966.5495.out cov.shardddl1.dmctl.1717054984.6088.out cov.shardddl1.dmctl.1717054987.6410.out cov.shardddl1.dmctl.1717054988.6447.out cov.shardddl1.dmctl.1717054989.6528.out cov.shardddl1.dmctl.1717054989.6669.out cov.shardddl1.dmctl.1717054990.6714.out cov.shardddl1.dmctl.1717054993.6833.out cov.shardddl1.dmctl.1717054993.6978.out cov.shardddl1.dmctl.1717054994.7018.out cov.shardddl1.dmctl.1717054995.7134.out cov.shardddl1.dmctl.1717054995.7170.out cov.shardddl1.dmctl.1717054995.7200.out cov.shardddl1.dmctl.1717054995.7235.out cov.shardddl1.dmctl.1717054996.7278.out cov.shardddl1.dmctl.1717055008.7595.out cov.shardddl1.dmctl.1717055014.7674.out cov.shardddl1.dmctl.1717055015.7708.out cov.shardddl1.dmctl.1717055015.7744.out cov.shardddl1.dmctl.1717055015.7887.out cov.shardddl1.dmctl.1717055016.7939.out cov.shardddl1.dmctl.1717055019.8137.out cov.shardddl1.dmctl.1717055019.8282.out cov.shardddl1.dmctl.1717055020.8319.out cov.shardddl1.dmctl.1717055021.8441.out cov.shardddl1.dmctl.1717055021.8476.out cov.shardddl1.dmctl.1717055021.8618.out cov.shardddl1.dmctl.1717055023.8653.out cov.shardddl1.dmctl.1717055023.8751.out cov.shardddl1.dmctl.1717055026.8886.out cov.shardddl1.dmctl.1717055026.8920.out cov.shardddl1.dmctl.1717055026.8954.out cov.shardddl1.dmctl.1717055031.9225.out cov.shardddl1.dmctl.1717055031.9278.out cov.shardddl1.dmctl.1717055033.9315.out cov.shardddl1.dmctl.1717055037.9453.out cov.shardddl1.dmctl.1717055038.9597.out cov.shardddl1.dmctl.1717055039.9631.out cov.shardddl1.dmctl.1717055043.9773.out cov.shardddl1.dmctl.1717055078.10062.out cov.shardddl1.dmctl.1717055078.10105.out cov.shardddl1.dmctl.1717055080.10154.out cov.shardddl1.dmctl.1717055082.10246.out cov.shardddl1.dmctl.1717055082.10282.out cov.shardddl1.dmctl.1717055089.10467.out cov.shardddl1.dmctl.1717055090.10505.out cov.shardddl1.dmctl.1717055090.10539.out cov.shardddl1.dmctl.1717055090.10680.out cov.shardddl1.dmctl.1717055091.10732.out cov.shardddl1.dmctl.1717055094.10830.out cov.shardddl1.dmctl.1717055094.10868.out cov.shardddl1.dmctl.1717055100.11060.out cov.shardddl1.dmctl.1717055100.11098.out cov.shardddl1.dmctl.1717055100.11137.out cov.shardddl1.dmctl.1717055101.11282.out cov.shardddl1.dmctl.1717055102.11324.out cov.shardddl1.dmctl.1717055111.11857.out cov.shardddl1.dmctl.1717055112.11999.out cov.shardddl1.dmctl.1717055113.12037.out cov.shardddl1.dmctl.1717055118.12484.out cov.shardddl1.dmctl.1717055118.12627.out cov.shardddl1.dmctl.1717055120.12661.out cov.shardddl1.dmctl.1717055122.12963.out cov.shardddl1.dmctl.1717055123.13106.out cov.shardddl1.dmctl.1717055124.13140.out cov.shardddl1.dmctl.1717055125.13403.out cov.shardddl1.dmctl.1717055128.13721.out cov.shardddl1.dmctl.1717055129.13761.out cov.shardddl1.dmctl.1717055132.13812.out cov.shardddl1.dmctl.1717055132.13847.out cov.shardddl1.dmctl.1717055135.14178.out cov.shardddl1.dmctl.1717055136.14231.out cov.shardddl1.dmctl.1717055137.14363.out cov.shardddl1.dmctl.1717055140.14696.out cov.shardddl1.dmctl.1717055142.14735.out cov.shardddl1.dmctl.1717055142.14779.out cov.shardddl1.dmctl.1717055142.14818.out cov.shardddl1.dmctl.1717055143.14957.out cov.shardddl1.dmctl.1717055145.14991.out cov.shardddl1.dmctl.1717055145.15063.out cov.shardddl1.dmctl.1717055146.15202.out cov.shardddl1.dmctl.1717055147.15243.out cov.shardddl1.dmctl.1717055155.15445.out cov.shardddl1.dmctl.1717055157.15589.out cov.shardddl1.dmctl.1717055158.15618.out cov.shardddl1.dmctl.1717055160.15721.out cov.shardddl1.dmctl.1717055162.15859.out cov.shardddl1.dmctl.1717055163.15894.out cov.shardddl1.dmctl.1717055163.15981.out cov.shardddl1.master.out cov.shardddl1.worker.8262.1717054893.out cov.shardddl1.worker.8262.1717054899.out cov.shardddl1.worker.8262.1717054928.out cov.shardddl1.worker.8262.1717054964.out cov.shardddl1.worker.8262.1717054986.out cov.shardddl1.worker.8262.1717055127.out cov.shardddl1.worker.8262.1717055134.out cov.shardddl1.worker.8262.1717055139.out cov.shardddl1.worker.8263.1717054895.out cov.shardddl1.worker.8263.1717054899.out cov.shardddl1.worker.8263.1717054928.out cov.shardddl1.worker.8263.1717054964.out cov.shardddl1.worker.8263.1717054986.out cov.shardddl1.worker.8263.1717055127.out cov.shardddl1.worker.8263.1717055134.out cov.shardddl1.worker.8263.1717055139.out cov.shardddl1_1.dmctl.1717055173.16435.out cov.shardddl1_1.dmctl.1717055175.16553.out cov.shardddl1_1.dmctl.1717055176.16645.out cov.shardddl1_1.dmctl.1717055177.16693.out cov.shardddl1_1.dmctl.1717055180.17037.out cov.shardddl1_1.dmctl.1717055180.17172.out cov.shardddl1_1.dmctl.1717055182.17201.out cov.shardddl1_1.dmctl.1717055182.17476.out cov.shardddl1_1.dmctl.1717055183.17622.out cov.shardddl1_1.dmctl.1717055184.17680.out cov.shardddl1_1.dmctl.1717055186.17798.out cov.shardddl1_1.dmctl.1717055188.17938.out cov.shardddl1_1.dmctl.1717055189.17985.out cov.shardddl1_1.dmctl.1717055191.18102.out cov.shardddl1_1.dmctl.1717055193.18242.out cov.shardddl1_1.dmctl.1717055194.18273.out cov.shardddl1_1.dmctl.1717055196.18400.out cov.shardddl1_1.dmctl.1717055198.18548.out cov.shardddl1_1.dmctl.1717055199.18577.out cov.shardddl1_1.dmctl.1717055201.18698.out cov.shardddl1_1.dmctl.1717055202.18838.out cov.shardddl1_1.dmctl.1717055204.18862.out cov.shardddl1_1.dmctl.1717055206.18997.out cov.shardddl1_1.dmctl.1717055207.19136.out cov.shardddl1_1.dmctl.1717055209.19171.out cov.shardddl1_1.dmctl.1717055209.19241.out cov.shardddl1_1.dmctl.1717055210.19382.out cov.shardddl1_1.dmctl.1717055212.19417.out cov.shardddl1_1.dmctl.1717055212.19484.out cov.shardddl1_1.dmctl.1717055213.19625.out cov.shardddl1_1.dmctl.1717055214.19652.out cov.shardddl1_1.dmctl.1717055215.19750.out cov.shardddl1_1.dmctl.1717055216.19892.out cov.shardddl1_1.dmctl.1717055217.19923.out cov.shardddl1_1.dmctl.1717055220.20003.out cov.shardddl1_1.dmctl.1717055221.20139.out cov.shardddl1_1.dmctl.1717055222.20171.out cov.shardddl1_1.dmctl.1717055224.20229.out cov.shardddl1_1.dmctl.1717055226.20364.out cov.shardddl1_1.dmctl.1717055227.20402.out cov.shardddl1_1.dmctl.1717055229.20457.out cov.shardddl1_1.dmctl.1717055231.20596.out cov.shardddl1_1.dmctl.1717055232.20625.out cov.shardddl1_1.dmctl.1717055234.20677.out cov.shardddl1_1.dmctl.1717055235.20813.out cov.shardddl1_1.dmctl.1717055237.20856.out cov.shardddl1_1.dmctl.1717055239.20912.out cov.shardddl1_1.dmctl.1717055240.21050.out cov.shardddl1_1.dmctl.1717055241.21081.out cov.shardddl1_1.dmctl.1717055241.21124.out cov.shardddl1_1.dmctl.1717055242.21256.out cov.shardddl1_1.dmctl.1717055243.21286.out cov.shardddl1_1.dmctl.1717055243.21327.out cov.shardddl1_1.dmctl.1717055244.21463.out cov.shardddl1_1.dmctl.1717055246.21495.out cov.shardddl1_1.dmctl.1717055248.21553.out cov.shardddl1_1.dmctl.1717055249.21694.out cov.shardddl1_1.dmctl.1717055250.21724.out cov.shardddl1_1.dmctl.1717055253.21774.out cov.shardddl1_1.dmctl.1717055254.21911.out cov.shardddl1_1.dmctl.1717055255.21944.out cov.shardddl1_1.dmctl.1717055257.21996.out cov.shardddl1_1.dmctl.1717055259.22140.out cov.shardddl1_1.dmctl.1717055261.22174.out cov.shardddl1_1.dmctl.1717055261.22207.out cov.shardddl1_1.dmctl.1717055261.22242.out cov.shardddl1_1.dmctl.1717055262.22385.out cov.shardddl1_1.dmctl.1717055264.22415.out cov.shardddl1_1.dmctl.1717055264.22507.out cov.shardddl1_1.dmctl.1717055265.22648.out cov.shardddl1_1.dmctl.1717055266.22682.out cov.shardddl1_1.dmctl.1717055267.22740.out cov.shardddl1_1.dmctl.1717055267.22775.out cov.shardddl1_1.dmctl.1717055268.22918.out cov.shardddl1_1.dmctl.1717055269.22948.out cov.shardddl1_1.dmctl.1717055269.22985.out cov.shardddl1_1.dmctl.1717055269.23017.out cov.shardddl1_1.dmctl.1717055271.23156.out cov.shardddl1_1.dmctl.1717055272.23186.out cov.shardddl1_1.dmctl.1717055274.23320.out cov.shardddl1_1.dmctl.1717055275.23460.out cov.shardddl1_1.dmctl.1717055276.23492.out cov.shardddl1_1.dmctl.1717055276.23588.out cov.shardddl1_1.dmctl.1717055278.23730.out cov.shardddl1_1.dmctl.1717055279.23765.out cov.shardddl1_1.dmctl.1717055279.23825.out cov.shardddl1_1.dmctl.1717055279.23966.out cov.shardddl1_1.dmctl.1717055281.23998.out cov.shardddl1_1.dmctl.1717055281.24056.out cov.shardddl1_1.dmctl.1717055281.24094.out cov.shardddl1_1.dmctl.1717055282.24241.out cov.shardddl1_1.dmctl.1717055284.24271.out cov.shardddl1_1.dmctl.1717055284.24400.out cov.shardddl1_1.dmctl.1717055285.24546.out cov.shardddl1_1.dmctl.1717055287.24576.out cov.shardddl1_1.dmctl.1717055289.24717.out cov.shardddl1_1.dmctl.1717055290.24861.out cov.shardddl1_1.dmctl.1717055292.24896.out cov.shardddl1_1.dmctl.1717055292.24960.out cov.shardddl1_1.dmctl.1717055292.24993.out cov.shardddl1_1.dmctl.1717055292.25137.out cov.shardddl1_1.dmctl.1717055293.25168.out cov.shardddl1_1.dmctl.1717055294.25233.out cov.shardddl1_1.dmctl.1717055294.25269.out cov.shardddl1_1.dmctl.1717055295.25417.out cov.shardddl1_1.dmctl.1717055296.25447.out cov.shardddl1_1.dmctl.1717055299.25553.out cov.shardddl1_1.master.out cov.shardddl1_1.worker.8262.1717055172.out cov.shardddl1_1.worker.8263.1717055174.out cov.shardddl2.dmctl.1717055309.26019.out cov.shardddl2.dmctl.1717055310.26146.out cov.shardddl2.dmctl.1717055311.26234.out cov.shardddl2.dmctl.1717055312.26308.out cov.shardddl2.dmctl.1717055318.26502.out cov.shardddl2.dmctl.1717055326.26697.out cov.shardddl2.dmctl.1717055331.26916.out cov.shardddl2.dmctl.1717055336.27060.out cov.shardddl2.dmctl.1717055336.27201.out cov.shardddl2.dmctl.1717055337.27238.out cov.shardddl2.dmctl.1717055347.27514.out cov.shardddl2.dmctl.1717055347.27575.out cov.shardddl2.dmctl.1717055347.27711.out cov.shardddl2.dmctl.1717055349.27756.out cov.shardddl2.dmctl.1717055354.27945.out cov.shardddl2.dmctl.1717055361.28148.out cov.shardddl2.dmctl.1717055364.28313.out cov.shardddl2.dmctl.1717055367.28416.out cov.shardddl2.dmctl.1717055367.28553.out cov.shardddl2.dmctl.1717055368.28611.out cov.shardddl2.dmctl.1717055388.29119.out cov.shardddl2.dmctl.1717055388.29150.out cov.shardddl2.dmctl.1717055389.29290.out cov.shardddl2.dmctl.1717055390.29332.out cov.shardddl2.dmctl.1717055392.29440.out cov.shardddl2.dmctl.1717055393.29491.out cov.shardddl2.dmctl.1717055393.29535.out cov.shardddl2.dmctl.1717055399.29708.out cov.shardddl2.dmctl.1717055399.29741.out cov.shardddl2.dmctl.1717055399.29774.out cov.shardddl2.dmctl.1717055400.29869.out cov.shardddl2.dmctl.1717055400.30007.out cov.shardddl2.dmctl.1717055401.30049.out cov.shardddl2.dmctl.1717055411.30303.out cov.shardddl2.dmctl.1717055411.30352.out cov.shardddl2.dmctl.1717055411.30391.out cov.shardddl2.dmctl.1717055418.30552.out cov.shardddl2.dmctl.1717055418.30591.out cov.shardddl2.dmctl.1717055418.30625.out cov.shardddl2.dmctl.1717055418.30712.out cov.shardddl2.dmctl.1717055418.30851.out cov.shardddl2.dmctl.1717055420.30904.out cov.shardddl2.dmctl.1717055426.31143.out cov.shardddl2.dmctl.1717055426.31192.out cov.shardddl2.dmctl.1717055427.31233.out cov.shardddl2.dmctl.1717055433.31393.out cov.shardddl2.dmctl.1717055433.31430.out cov.shardddl2.dmctl.1717055433.31465.out cov.shardddl2.dmctl.1717055434.31560.out cov.shardddl2.dmctl.1717055434.31699.out cov.shardddl2.dmctl.1717055435.31748.out cov.shardddl2.dmctl.1717055438.31854.out cov.shardddl2.dmctl.1717055447.32058.out cov.shardddl2.dmctl.1717055447.32099.out cov.shardddl2.dmctl.1717055454.32248.out cov.shardddl2.dmctl.1717055454.32285.out cov.shardddl2.dmctl.1717055454.32319.out cov.shardddl2.dmctl.1717055454.32409.out cov.shardddl2.dmctl.1717055455.32549.out cov.shardddl2.dmctl.1717055456.32590.out cov.shardddl2.dmctl.1717055458.32700.out cov.shardddl2.dmctl.1717055466.32892.out cov.shardddl2.dmctl.1717055466.32935.out cov.shardddl2.dmctl.1717055474.33086.out cov.shardddl2.dmctl.1717055474.33123.out cov.shardddl2.dmctl.1717055474.33158.out cov.shardddl2.dmctl.1717055474.33260.out cov.shardddl2.dmctl.1717055475.33398.out cov.shardddl2.dmctl.1717055476.33439.out cov.shardddl2.dmctl.1717055478.33540.out cov.shardddl2.dmctl.1717055478.33593.out cov.shardddl2.dmctl.1717055485.33750.out cov.shardddl2.master.out cov.shardddl2.worker.8262.1717055308.out cov.shardddl2.worker.8262.1717055315.out cov.shardddl2.worker.8263.1717055309.out cov.shardddl2.worker.8263.1717055351.out downstream goroutines shardddl1 shardddl1_1 shardddl2 sql_res.shardddl1.txt sql_res.shardddl1_1.txt sql_res.shardddl2.txt tidb.toml ++ find /tmp/dm_test/ -type f -name '*.log' + tar -cvzf log-G07.tar.gz /tmp/dm_test/shardddl1/dmctl.1717055123.log /tmp/dm_test/shardddl1/dmctl.1717055145.log /tmp/dm_test/shardddl1/dmctl.1717055118.log /tmp/dm_test/shardddl1/dmctl.1717054962.log /tmp/dm_test/shardddl1/dmctl.1717055158.log /tmp/dm_test/shardddl1/dmctl.1717054900.log /tmp/dm_test/shardddl1/dmctl.1717055101.log /tmp/dm_test/shardddl1/dmctl.1717055019.log /tmp/dm_test/shardddl1/dmctl.1717054990.log /tmp/dm_test/shardddl1/dmctl.1717055160.log /tmp/dm_test/shardddl1/dmctl.1717055008.log /tmp/dm_test/shardddl1/dmctl.1717055021.log /tmp/dm_test/shardddl1/dmctl.1717055111.log /tmp/dm_test/shardddl1/dmctl.1717054996.log /tmp/dm_test/shardddl1/dmctl.1717055147.log /tmp/dm_test/shardddl1/dmctl.1717054988.log /tmp/dm_test/shardddl1/dmctl.1717054995.log /tmp/dm_test/shardddl1/dmctl.1717055129.log /tmp/dm_test/shardddl1/dmctl.1717055082.log /tmp/dm_test/shardddl1/dmctl.1717055142.log /tmp/dm_test/shardddl1/worker1/log/stdout.log /tmp/dm_test/shardddl1/worker1/log/dm-worker.log /tmp/dm_test/shardddl1/dmctl.1717055023.log /tmp/dm_test/shardddl1/dmctl.1717054966.log /tmp/dm_test/shardddl1/dmctl.1717054994.log /tmp/dm_test/shardddl1/dmctl.1717054894.log /tmp/dm_test/shardddl1/dmctl.1717055020.log /tmp/dm_test/shardddl1/dmctl.1717055113.log /tmp/dm_test/shardddl1/dmctl.1717055163.log /tmp/dm_test/shardddl1/dmctl.1717055136.log /tmp/dm_test/shardddl1/dmctl.1717055162.log /tmp/dm_test/shardddl1/dmctl.1717055089.log /tmp/dm_test/shardddl1/dmctl.1717054965.log /tmp/dm_test/shardddl1/dmctl.1717055037.log /tmp/dm_test/shardddl1/dmctl.1717055140.log /tmp/dm_test/shardddl1/dmctl.1717055033.log /tmp/dm_test/shardddl1/dmctl.1717055014.log /tmp/dm_test/shardddl1/dmctl.1717055039.log /tmp/dm_test/shardddl1/dmctl.1717054901.log /tmp/dm_test/shardddl1/dmctl.1717055137.log /tmp/dm_test/shardddl1/dmctl.1717055157.log /tmp/dm_test/shardddl1/dmctl.1717055122.log /tmp/dm_test/shardddl1/dmctl.1717055094.log /tmp/dm_test/shardddl1/dmctl.1717055100.log /tmp/dm_test/shardddl1/dmctl.1717055124.log /tmp/dm_test/shardddl1/dmctl.1717055078.log /tmp/dm_test/shardddl1/dmctl.1717054931.log /tmp/dm_test/shardddl1/dmctl.1717054896.log /tmp/dm_test/shardddl1/dmctl.1717054929.log /tmp/dm_test/shardddl1/dmctl.1717055135.log /tmp/dm_test/shardddl1/dmctl.1717055143.log /tmp/dm_test/shardddl1/dmctl.1717055091.log /tmp/dm_test/shardddl1/dmctl.1717055080.log /tmp/dm_test/shardddl1/dmctl.1717054987.log /tmp/dm_test/shardddl1/dmctl.1717054993.log /tmp/dm_test/shardddl1/dmctl.1717054984.log /tmp/dm_test/shardddl1/dmctl.1717055120.log /tmp/dm_test/shardddl1/dmctl.1717055016.log /tmp/dm_test/shardddl1/dmctl.1717055038.log /tmp/dm_test/shardddl1/dmctl.1717055031.log /tmp/dm_test/shardddl1/dmctl.1717055155.log /tmp/dm_test/shardddl1/dmctl.1717055146.log /tmp/dm_test/shardddl1/dmctl.1717055026.log /tmp/dm_test/shardddl1/dmctl.1717055102.log /tmp/dm_test/shardddl1/worker2/log/stdout.log /tmp/dm_test/shardddl1/worker2/log/dm-worker.log /tmp/dm_test/shardddl1/dmctl.1717055112.log /tmp/dm_test/shardddl1/dmctl.1717055125.log /tmp/dm_test/shardddl1/sync_diff_stdout.log /tmp/dm_test/shardddl1/dmctl.1717055043.log /tmp/dm_test/shardddl1/dmctl.1717055015.log /tmp/dm_test/shardddl1/dmctl.1717054989.log /tmp/dm_test/shardddl1/dmctl.1717055128.log /tmp/dm_test/shardddl1/dmctl.1717055090.log /tmp/dm_test/shardddl1/dmctl.1717054926.log /tmp/dm_test/shardddl1/dmctl.1717055132.log /tmp/dm_test/shardddl1/master/log/stdout.log /tmp/dm_test/shardddl1/master/log/dm-master.log /tmp/dm_test/shardddl1_1/dmctl.1717055191.log /tmp/dm_test/shardddl1_1/dmctl.1717055199.log /tmp/dm_test/shardddl1_1/dmctl.1717055243.log /tmp/dm_test/shardddl1_1/dmctl.1717055188.log /tmp/dm_test/shardddl1_1/dmctl.1717055217.log /tmp/dm_test/shardddl1_1/dmctl.1717055276.log /tmp/dm_test/shardddl1_1/dmctl.1717055244.log /tmp/dm_test/shardddl1_1/dmctl.1717055196.log /tmp/dm_test/shardddl1_1/dmctl.1717055274.log /tmp/dm_test/shardddl1_1/dmctl.1717055272.log /tmp/dm_test/shardddl1_1/dmctl.1717055246.log /tmp/dm_test/shardddl1_1/dmctl.1717055206.log /tmp/dm_test/shardddl1_1/dmctl.1717055255.log /tmp/dm_test/shardddl1_1/dmctl.1717055216.log /tmp/dm_test/shardddl1_1/dmctl.1717055213.log /tmp/dm_test/shardddl1_1/dmctl.1717055186.log /tmp/dm_test/shardddl1_1/dmctl.1717055202.log /tmp/dm_test/shardddl1_1/dmctl.1717055299.log /tmp/dm_test/shardddl1_1/dmctl.1717055269.log /tmp/dm_test/shardddl1_1/dmctl.1717055295.log /tmp/dm_test/shardddl1_1/dmctl.1717055229.log /tmp/dm_test/shardddl1_1/dmctl.1717055224.log /tmp/dm_test/shardddl1_1/dmctl.1717055293.log /tmp/dm_test/shardddl1_1/dmctl.1717055175.log /tmp/dm_test/shardddl1_1/worker1/log/stdout.log /tmp/dm_test/shardddl1_1/worker1/log/dm-worker.log /tmp/dm_test/shardddl1_1/dmctl.1717055189.log /tmp/dm_test/shardddl1_1/dmctl.1717055193.log /tmp/dm_test/shardddl1_1/dmctl.1717055214.log /tmp/dm_test/shardddl1_1/dmctl.1717055241.log /tmp/dm_test/shardddl1_1/dmctl.1717055257.log /tmp/dm_test/shardddl1_1/dmctl.1717055212.log /tmp/dm_test/shardddl1_1/dmctl.1717055289.log /tmp/dm_test/shardddl1_1/dmctl.1717055184.log /tmp/dm_test/shardddl1_1/dmctl.1717055292.log /tmp/dm_test/shardddl1_1/dmctl.1717055281.log /tmp/dm_test/shardddl1_1/dmctl.1717055201.log /tmp/dm_test/shardddl1_1/dmctl.1717055232.log /tmp/dm_test/shardddl1_1/dmctl.1717055227.log /tmp/dm_test/shardddl1_1/dmctl.1717055182.log /tmp/dm_test/shardddl1_1/dmctl.1717055240.log /tmp/dm_test/shardddl1_1/dmctl.1717055222.log /tmp/dm_test/shardddl1_1/dmctl.1717055173.log /tmp/dm_test/shardddl1_1/dmctl.1717055268.log /tmp/dm_test/shardddl1_1/dmctl.1717055265.log /tmp/dm_test/shardddl1_1/dmctl.1717055278.log /tmp/dm_test/shardddl1_1/dmctl.1717055226.log /tmp/dm_test/shardddl1_1/dmctl.1717055290.log /tmp/dm_test/shardddl1_1/dmctl.1717055235.log /tmp/dm_test/shardddl1_1/dmctl.1717055275.log /tmp/dm_test/shardddl1_1/dmctl.1717055204.log /tmp/dm_test/shardddl1_1/dmctl.1717055285.log /tmp/dm_test/shardddl1_1/dmctl.1717055242.log /tmp/dm_test/shardddl1_1/dmctl.1717055234.log /tmp/dm_test/shardddl1_1/dmctl.1717055253.log /tmp/dm_test/shardddl1_1/dmctl.1717055198.log /tmp/dm_test/shardddl1_1/dmctl.1717055267.log /tmp/dm_test/shardddl1_1/dmctl.1717055294.log /tmp/dm_test/shardddl1_1/dmctl.1717055259.log /tmp/dm_test/shardddl1_1/dmctl.1717055262.log /tmp/dm_test/shardddl1_1/dmctl.1717055248.log /tmp/dm_test/shardddl1_1/dmctl.1717055183.log /tmp/dm_test/shardddl1_1/dmctl.1717055282.log /tmp/dm_test/shardddl1_1/dmctl.1717055249.log /tmp/dm_test/shardddl1_1/dmctl.1717055237.log /tmp/dm_test/shardddl1_1/dmctl.1717055231.log /tmp/dm_test/shardddl1_1/dmctl.1717055177.log /tmp/dm_test/shardddl1_1/dmctl.1717055266.log /tmp/dm_test/shardddl1_1/dmctl.1717055279.log /tmp/dm_test/shardddl1_1/dmctl.1717055254.log /tmp/dm_test/shardddl1_1/dmctl.1717055210.log /tmp/dm_test/shardddl1_1/dmctl.1717055239.log /tmp/dm_test/shardddl1_1/dmctl.1717055296.log /tmp/dm_test/shardddl1_1/dmctl.1717055209.log /tmp/dm_test/shardddl1_1/dmctl.1717055180.log /tmp/dm_test/shardddl1_1/dmctl.1717055215.log /tmp/dm_test/shardddl1_1/dmctl.1717055250.log /tmp/dm_test/shardddl1_1/worker2/log/stdout.log /tmp/dm_test/shardddl1_1/worker2/log/dm-worker.log /tmp/dm_test/shardddl1_1/dmctl.1717055287.log /tmp/dm_test/shardddl1_1/dmctl.1717055194.log /tmp/dm_test/shardddl1_1/sync_diff_stdout.log /tmp/dm_test/shardddl1_1/dmctl.1717055264.log /tmp/dm_test/shardddl1_1/dmctl.1717055271.log /tmp/dm_test/shardddl1_1/dmctl.1717055207.log /tmp/dm_test/shardddl1_1/dmctl.1717055261.log /tmp/dm_test/shardddl1_1/dmctl.1717055220.log /tmp/dm_test/shardddl1_1/dmctl.1717055284.log /tmp/dm_test/shardddl1_1/dmctl.1717055221.log /tmp/dm_test/shardddl1_1/dmctl.1717055176.log /tmp/dm_test/shardddl1_1/master/log/stdout.log /tmp/dm_test/shardddl1_1/master/log/dm-master.log /tmp/dm_test/shardddl2/dmctl.1717055392.log /tmp/dm_test/shardddl2/dmctl.1717055309.log /tmp/dm_test/shardddl2/dmctl.1717055347.log /tmp/dm_test/shardddl2/dmctl.1717055420.log /tmp/dm_test/shardddl2/dmctl.1717055400.log /tmp/dm_test/shardddl2/dmctl.1717055454.log /tmp/dm_test/shardddl2/dmctl.1717055401.log /tmp/dm_test/shardddl2/dmctl.1717055311.log /tmp/dm_test/shardddl2/dmctl.1717055336.log /tmp/dm_test/shardddl2/dmctl.1717055418.log /tmp/dm_test/shardddl2/dmctl.1717055393.log /tmp/dm_test/shardddl2/dmctl.1717055326.log /tmp/dm_test/shardddl2/worker1/log/stdout.log /tmp/dm_test/shardddl2/worker1/log/dm-worker.log /tmp/dm_test/shardddl2/dmctl.1717055485.log /tmp/dm_test/shardddl2/dmctl.1717055433.log /tmp/dm_test/shardddl2/dmctl.1717055466.log /tmp/dm_test/shardddl2/dmctl.1717055367.log /tmp/dm_test/shardddl2/dmctl.1717055435.log /tmp/dm_test/shardddl2/dmctl.1717055474.log /tmp/dm_test/shardddl2/dmctl.1717055427.log /tmp/dm_test/shardddl2/dmctl.1717055411.log /tmp/dm_test/shardddl2/dmctl.1717055361.log /tmp/dm_test/shardddl2/dmctl.1717055354.log /tmp/dm_test/shardddl2/dmctl.1717055364.log /tmp/dm_test/shardddl2/dmctl.1717055447.log /tmp/dm_test/shardddl2/dmctl.1717055458.log /tmp/dm_test/shardddl2/dmctl.1717055312.log /tmp/dm_test/shardddl2/dmctl.1717055318.log /tmp/dm_test/shardddl2/dmctl.1717055390.log /tmp/dm_test/shardddl2/dmctl.1717055426.log /tmp/dm_test/shardddl2/dmctl.1717055455.log /tmp/dm_test/shardddl2/dmctl.1717055475.log /tmp/dm_test/shardddl2/dmctl.1717055310.log /tmp/dm_test/shardddl2/worker2/log/stdout.log /tmp/dm_test/shardddl2/worker2/log/dm-worker.log /tmp/dm_test/shardddl2/dmctl.1717055349.log /tmp/dm_test/shardddl2/dmctl.1717055399.log /tmp/dm_test/shardddl2/sync_diff_stdout.log /tmp/dm_test/shardddl2/dmctl.1717055389.log /tmp/dm_test/shardddl2/dmctl.1717055388.log /tmp/dm_test/shardddl2/dmctl.1717055434.log /tmp/dm_test/shardddl2/dmctl.1717055456.log /tmp/dm_test/shardddl2/dmctl.1717055476.log /tmp/dm_test/shardddl2/dmctl.1717055368.log /tmp/dm_test/shardddl2/dmctl.1717055438.log /tmp/dm_test/shardddl2/dmctl.1717055331.log /tmp/dm_test/shardddl2/dmctl.1717055478.log /tmp/dm_test/shardddl2/dmctl.1717055337.log /tmp/dm_test/shardddl2/master/log/stdout.log /tmp/dm_test/shardddl2/master/log/dm-master.log /tmp/dm_test/downstream/tidb/log/tidb.log /tmp/dm_test/goroutines/stack/log/worker-8264.log /tmp/dm_test/goroutines/stack/log/master-8561.log /tmp/dm_test/goroutines/stack/log/worker-18262.log /tmp/dm_test/goroutines/stack/log/master-8461.log /tmp/dm_test/goroutines/stack/log/master-8761.log /tmp/dm_test/goroutines/stack/log/worker-8262.log /tmp/dm_test/goroutines/stack/log/master-8261.log /tmp/dm_test/goroutines/stack/log/worker-8263.log /tmp/dm_test/goroutines/stack/log/worker-18263.log /tmp/dm_test/goroutines/stack/log/master-8661.log /tmp/dm_test/goroutines/stack/log/master-8361.log tar: Removing leading `/' from member names /tmp/dm_test/shardddl1/dmctl.1717055123.log /tmp/dm_test/shardddl1/dmctl.1717055145.log /tmp/dm_test/shardddl1/dmctl.1717055118.log /tmp/dm_test/shardddl1/dmctl.1717054962.log /tmp/dm_test/shardddl1/dmctl.1717055158.log /tmp/dm_test/shardddl1/dmctl.1717054900.log /tmp/dm_test/shardddl1/dmctl.1717055101.log /tmp/dm_test/shardddl1/dmctl.1717055019.log /tmp/dm_test/shardddl1/dmctl.1717054990.log /tmp/dm_test/shardddl1/dmctl.1717055160.log /tmp/dm_test/shardddl1/dmctl.1717055008.log /tmp/dm_test/shardddl1/dmctl.1717055021.log /tmp/dm_test/shardddl1/dmctl.1717055111.log /tmp/dm_test/shardddl1/dmctl.1717054996.log /tmp/dm_test/shardddl1/dmctl.1717055147.log /tmp/dm_test/shardddl1/dmctl.1717054988.log /tmp/dm_test/shardddl1/dmctl.1717054995.log /tmp/dm_test/shardddl1/dmctl.1717055129.log /tmp/dm_test/shardddl1/dmctl.1717055082.log /tmp/dm_test/shardddl1/dmctl.1717055142.log /tmp/dm_test/shardddl1/worker1/log/stdout.log /tmp/dm_test/shardddl1/worker1/log/dm-worker.log /tmp/dm_test/shardddl1/dmctl.1717055023.log /tmp/dm_test/shardddl1/dmctl.1717054966.log /tmp/dm_test/shardddl1/dmctl.1717054994.log /tmp/dm_test/shardddl1/dmctl.1717054894.log /tmp/dm_test/shardddl1/dmctl.1717055020.log /tmp/dm_test/shardddl1/dmctl.1717055113.log /tmp/dm_test/shardddl1/dmctl.1717055163.log /tmp/dm_test/shardddl1/dmctl.1717055136.log /tmp/dm_test/shardddl1/dmctl.1717055162.log /tmp/dm_test/shardddl1/dmctl.1717055089.log /tmp/dm_test/shardddl1/dmctl.1717054965.log /tmp/dm_test/shardddl1/dmctl.1717055037.log /tmp/dm_test/shardddl1/dmctl.1717055140.log /tmp/dm_test/shardddl1/dmctl.1717055033.log /tmp/dm_test/shardddl1/dmctl.1717055014.log /tmp/dm_test/shardddl1/dmctl.1717055039.log /tmp/dm_test/shardddl1/dmctl.1717054901.log /tmp/dm_test/shardddl1/dmctl.1717055137.log /tmp/dm_test/shardddl1/dmctl.1717055157.log /tmp/dm_test/shardddl1/dmctl.1717055122.log /tmp/dm_test/shardddl1/dmctl.1717055094.log /tmp/dm_test/shardddl1/dmctl.1717055100.log /tmp/dm_test/shardddl1/dmctl.1717055124.log /tmp/dm_test/shardddl1/dmctl.1717055078.log /tmp/dm_test/shardddl1/dmctl.1717054931.log /tmp/dm_test/shardddl1/dmctl.1717054896.log /tmp/dm_test/shardddl1/dmctl.1717054929.log /tmp/dm_test/shardddl1/dmctl.1717055135.log /tmp/dm_test/shardddl1/dmctl.1717055143.log /tmp/dm_test/shardddl1/dmctl.1717055091.log /tmp/dm_test/shardddl1/dmctl.1717055080.log /tmp/dm_test/shardddl1/dmctl.1717054987.log /tmp/dm_test/shardddl1/dmctl.1717054993.log /tmp/dm_test/shardddl1/dmctl.1717054984.log /tmp/dm_test/shardddl1/dmctl.1717055120.log /tmp/dm_test/shardddl1/dmctl.1717055016.log /tmp/dm_test/shardddl1/dmctl.1717055038.log /tmp/dm_test/shardddl1/dmctl.1717055031.log /tmp/dm_test/shardddl1/dmctl.1717055155.log /tmp/dm_test/shardddl1/dmctl.1717055146.log /tmp/dm_test/shardddl1/dmctl.1717055026.log /tmp/dm_test/shardddl1/dmctl.1717055102.log /tmp/dm_test/shardddl1/worker2/log/stdout.log /tmp/dm_test/shardddl1/worker2/log/dm-worker.log /tmp/dm_test/shardddl1/dmctl.1717055112.log /tmp/dm_test/shardddl1/dmctl.1717055125.log /tmp/dm_test/shardddl1/sync_diff_stdout.log /tmp/dm_test/shardddl1/dmctl.1717055043.log /tmp/dm_test/shardddl1/dmctl.1717055015.log /tmp/dm_test/shardddl1/dmctl.1717054989.log /tmp/dm_test/shardddl1/dmctl.1717055128.log /tmp/dm_test/shardddl1/dmctl.1717055090.log /tmp/dm_test/shardddl1/dmctl.1717054926.log /tmp/dm_test/shardddl1/dmctl.1717055132.log /tmp/dm_test/shardddl1/master/log/stdout.log /tmp/dm_test/shardddl1/master/log/dm-master.log /tmp/dm_test/shardddl1_1/dmctl.1717055191.log /tmp/dm_test/shardddl1_1/dmctl.1717055199.log /tmp/dm_test/shardddl1_1/dmctl.1717055243.log /tmp/dm_test/shardddl1_1/dmctl.1717055188.log /tmp/dm_test/shardddl1_1/dmctl.1717055217.log /tmp/dm_test/shardddl1_1/dmctl.1717055276.log /tmp/dm_test/shardddl1_1/dmctl.1717055244.log /tmp/dm_test/shardddl1_1/dmctl.1717055196.log /tmp/dm_test/shardddl1_1/dmctl.1717055274.log /tmp/dm_test/shardddl1_1/dmctl.1717055272.log /tmp/dm_test/shardddl1_1/dmctl.1717055246.log /tmp/dm_test/shardddl1_1/dmctl.1717055206.log /tmp/dm_test/shardddl1_1/dmctl.1717055255.log /tmp/dm_test/shardddl1_1/dmctl.1717055216.log /tmp/dm_test/shardddl1_1/dmctl.1717055213.log /tmp/dm_test/shardddl1_1/dmctl.1717055186.log /tmp/dm_test/shardddl1_1/dmctl.1717055202.log /tmp/dm_test/shardddl1_1/dmctl.1717055299.log /tmp/dm_test/shardddl1_1/dmctl.1717055269.log /tmp/dm_test/shardddl1_1/dmctl.1717055295.log /tmp/dm_test/shardddl1_1/dmctl.1717055229.log /tmp/dm_test/shardddl1_1/dmctl.1717055224.log /tmp/dm_test/shardddl1_1/dmctl.1717055293.log /tmp/dm_test/shardddl1_1/dmctl.1717055175.log /tmp/dm_test/shardddl1_1/worker1/log/stdout.log /tmp/dm_test/shardddl1_1/worker1/log/dm-worker.log /tmp/dm_test/shardddl1_1/dmctl.1717055189.log /tmp/dm_test/shardddl1_1/dmctl.1717055193.log /tmp/dm_test/shardddl1_1/dmctl.1717055214.log /tmp/dm_test/shardddl1_1/dmctl.1717055241.log /tmp/dm_test/shardddl1_1/dmctl.1717055257.log /tmp/dm_test/shardddl1_1/dmctl.1717055212.log /tmp/dm_test/shardddl1_1/dmctl.1717055289.log /tmp/dm_test/shardddl1_1/dmctl.1717055184.log /tmp/dm_test/shardddl1_1/dmctl.1717055292.log /tmp/dm_test/shardddl1_1/dmctl.1717055281.log /tmp/dm_test/shardddl1_1/dmctl.1717055201.log /tmp/dm_test/shardddl1_1/dmctl.1717055232.log /tmp/dm_test/shardddl1_1/dmctl.1717055227.log /tmp/dm_test/shardddl1_1/dmctl.1717055182.log /tmp/dm_test/shardddl1_1/dmctl.1717055240.log /tmp/dm_test/shardddl1_1/dmctl.1717055222.log /tmp/dm_test/shardddl1_1/dmctl.1717055173.log /tmp/dm_test/shardddl1_1/dmctl.1717055268.log /tmp/dm_test/shardddl1_1/dmctl.1717055265.log /tmp/dm_test/shardddl1_1/dmctl.1717055278.log /tmp/dm_test/shardddl1_1/dmctl.1717055226.log /tmp/dm_test/shardddl1_1/dmctl.1717055290.log /tmp/dm_test/shardddl1_1/dmctl.1717055235.log /tmp/dm_test/shardddl1_1/dmctl.1717055275.log /tmp/dm_test/shardddl1_1/dmctl.1717055204.log /tmp/dm_test/shardddl1_1/dmctl.1717055285.log /tmp/dm_test/shardddl1_1/dmctl.1717055242.log /tmp/dm_test/shardddl1_1/dmctl.1717055234.log /tmp/dm_test/shardddl1_1/dmctl.1717055253.log /tmp/dm_test/shardddl1_1/dmctl.1717055198.log /tmp/dm_test/shardddl1_1/dmctl.1717055267.log /tmp/dm_test/shardddl1_1/dmctl.1717055294.log /tmp/dm_test/shardddl1_1/dmctl.1717055259.log /tmp/dm_test/shardddl1_1/dmctl.1717055262.log /tmp/dm_test/shardddl1_1/dmctl.1717055248.log /tmp/dm_test/shardddl1_1/dmctl.1717055183.log /tmp/dm_test/shardddl1_1/dmctl.1717055282.log /tmp/dm_test/shardddl1_1/dmctl.1717055249.log /tmp/dm_test/shardddl1_1/dmctl.1717055237.log /tmp/dm_test/shardddl1_1/dmctl.1717055231.log /tmp/dm_test/shardddl1_1/dmctl.1717055177.log /tmp/dm_test/shardddl1_1/dmctl.1717055266.log /tmp/dm_test/shardddl1_1/dmctl.1717055279.log /tmp/dm_test/shardddl1_1/dmctl.1717055254.log /tmp/dm_test/shardddl1_1/dmctl.1717055210.log /tmp/dm_test/shardddl1_1/dmctl.1717055239.log /tmp/dm_test/shardddl1_1/dmctl.1717055296.log /tmp/dm_test/shardddl1_1/dmctl.1717055209.log /tmp/dm_test/shardddl1_1/dmctl.1717055180.log /tmp/dm_test/shardddl1_1/dmctl.1717055215.log /tmp/dm_test/shardddl1_1/dmctl.1717055250.log /tmp/dm_test/shardddl1_1/worker2/log/stdout.log /tmp/dm_test/shardddl1_1/worker2/log/dm-worker.log /tmp/dm_test/shardddl1_1/dmctl.1717055287.log /tmp/dm_test/shardddl1_1/dmctl.1717055194.log /tmp/dm_test/shardddl1_1/sync_diff_stdout.log /tmp/dm_test/shardddl1_1/dmctl.1717055264.log /tmp/dm_test/shardddl1_1/dmctl.1717055271.log /tmp/dm_test/shardddl1_1/dmctl.1717055207.log /tmp/dm_test/shardddl1_1/dmctl.1717055261.log /tmp/dm_test/shardddl1_1/dmctl.1717055220.log /tmp/dm_test/shardddl1_1/dmctl.1717055284.log /tmp/dm_test/shardddl1_1/dmctl.1717055221.log /tmp/dm_test/shardddl1_1/dmctl.1717055176.log /tmp/dm_test/shardddl1_1/master/log/stdout.log /tmp/dm_test/shardddl1_1/master/log/dm-master.log /tmp/dm_test/shardddl2/dmctl.1717055392.log /tmp/dm_test/shardddl2/dmctl.1717055309.log /tmp/dm_test/shardddl2/dmctl.1717055347.log /tmp/dm_test/shardddl2/dmctl.1717055420.log /tmp/dm_test/shardddl2/dmctl.1717055400.log /tmp/dm_test/shardddl2/dmctl.1717055454.log /tmp/dm_test/shardddl2/dmctl.1717055401.log /tmp/dm_test/shardddl2/dmctl.1717055311.log /tmp/dm_test/shardddl2/dmctl.1717055336.log /tmp/dm_test/shardddl2/dmctl.1717055418.log /tmp/dm_test/shardddl2/dmctl.1717055393.log /tmp/dm_test/shardddl2/dmctl.1717055326.log /tmp/dm_test/shardddl2/worker1/log/stdout.log /tmp/dm_test/shardddl2/worker1/log/dm-worker.log /tmp/dm_test/shardddl2/dmctl.1717055485.log /tmp/dm_test/shardddl2/dmctl.1717055433.log /tmp/dm_test/shardddl2/dmctl.1717055466.log /tmp/dm_test/shardddl2/dmctl.1717055367.log /tmp/dm_test/shardddl2/dmctl.1717055435.log /tmp/dm_test/shardddl2/dmctl.1717055474.log /tmp/dm_test/shardddl2/dmctl.1717055427.log /tmp/dm_test/shardddl2/dmctl.1717055411.log /tmp/dm_test/shardddl2/dmctl.1717055361.log /tmp/dm_test/shardddl2/dmctl.1717055354.log /tmp/dm_test/shardddl2/dmctl.1717055364.log /tmp/dm_test/shardddl2/dmctl.1717055447.log /tmp/dm_test/shardddl2/dmctl.1717055458.log /tmp/dm_test/shardddl2/dmctl.1717055312.log /tmp/dm_test/shardddl2/dmctl.1717055318.log /tmp/dm_test/shardddl2/dmctl.1717055390.log /tmp/dm_test/shardddl2/dmctl.1717055426.log /tmp/dm_test/shardddl2/dmctl.1717055455.log /tmp/dm_test/shardddl2/dmctl.1717055475.log /tmp/dm_test/shardddl2/dmctl.1717055310.log /tmp/dm_test/shardddl2/worker2/log/stdout.log /tmp/dm_test/shardddl2/worker2/log/dm-worker.log /tmp/dm_test/shardddl2/dmctl.1717055349.log /tmp/dm_test/shardddl2/dmctl.1717055399.log /tmp/dm_test/shardddl2/sync_diff_stdout.log /tmp/dm_test/shardddl2/dmctl.1717055389.log /tmp/dm_test/shardddl2/dmctl.1717055388.log /tmp/dm_test/shardddl2/dmctl.1717055434.log /tmp/dm_test/shardddl2/dmctl.1717055456.log /tmp/dm_test/shardddl2/dmctl.1717055476.log /tmp/dm_test/shardddl2/dmctl.1717055368.log /tmp/dm_test/shardddl2/dmctl.1717055438.log /tmp/dm_test/shardddl2/dmctl.1717055331.log /tmp/dm_test/shardddl2/dmctl.1717055478.log /tmp/dm_test/shardddl2/dmctl.1717055337.log /tmp/dm_test/shardddl2/master/log/stdout.log /tmp/dm_test/shardddl2/master/log/dm-master.log /tmp/dm_test/downstream/tidb/log/tidb.log /tmp/dm_test/goroutines/stack/log/worker-8264.log /tmp/dm_test/goroutines/stack/log/master-8561.log /tmp/dm_test/goroutines/stack/log/worker-18262.log /tmp/dm_test/goroutines/stack/log/master-8461.log /tmp/dm_test/goroutines/stack/log/master-8761.log /tmp/dm_test/goroutines/stack/log/worker-8262.log /tmp/dm_test/goroutines/stack/log/master-8261.log /tmp/dm_test/goroutines/stack/log/worker-8263.log /tmp/dm_test/goroutines/stack/log/worker-18263.log /tmp/dm_test/goroutines/stack/log/master-8661.log /tmp/dm_test/goroutines/stack/log/master-8361.log + ls -alh log-G07.tar.gz -rw-r--r-- 1 jenkins jenkins 672K May 30 15:51 log-G07.tar.gz [Pipeline] archiveArtifacts Archiving artifacts [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage rpc addr 127.0.0.1:8261 is alive [Thu May 30 15:51:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/foreign_key/conf/dm-worker1.toml >>>>>> [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G07' [Thu May 30 15:51:49 CST 2024] <<<<<< start DM-134 pessimistic >>>>>> Sending interrupt signal to process Killing processes wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta" kill finished with exit code 0 Sending interrupt signal to process Killing processes make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 2 Finished: FAILURE