Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_dm_integration_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h Agent pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h is provisioned from template pingcap_tiflow_pull_dm_integration_test_1875-7mhv2-sj1t3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "298aa090069191084272f354a9efb5453a848eb6" jenkins/label: "pingcap_tiflow_pull_dm_integration_test_1875-7mhv2" name: "pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=05e76d82-39db-4e0e-b7c2-10a97931fef6 JENKINS_NODE_COOKIE=9a533cba-34b5-4799-a93c-8aa8699f060f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_dm_integration_test-1875 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_dm_integration_test","buildid":"1786658626056753152","prowjobid":"05e76d82-39db-4e0e-b7c2-10a97931fef6","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7dd6241bd56139ff08ada1b98fc467877b87f31a","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7dd6241bd56139ff08ada1b98fc467877b87f31a","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_dm_integration_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1875 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786658626056753152 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_dm_integration_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_dm_integration_test_1875-7mhv2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-e4960c3787c8b0a028d56044159150f6020eb6e83c507357bbc62d29b344724b NODE_LABELS=pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h pingcap_tiflow_pull_dm_integration_test_1875-7mhv2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/display/redirect BUILD_NUMBER=1875 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3559055939=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h -c golang -- bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-dm-integration-test-1875-7mhv2-sj1t3-c6b0h -c golang -- bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 15581 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: a377ad633837b3bc (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sat May 04 07:26:57 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, tests/integration_tests/_utils/check_sync_diff, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/open_protocol_handle_key_only/data/data.sql, tests/integration_tests/open_protocol_handle_key_only/run.sh, tests/integration_tests/run_group.sh] [Pipeline] echo diff file matched: pkg/applier/redo.go [Pipeline] echo matched, some diff files full path start with dm/ or pkg/ or go.mod, run the dm integration test [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534) 203635712 bytes in 2.61 secs (77963931 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/.git/ .git HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) POST git-upload-pack (656 bytes) POST git-upload-pack (973 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master * [new ref] refs/pull/10919/head -> origin/pr/10919/head HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..7dd6241bd Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 35 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 89 +++--- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 303 +++++++++++++++++- pkg/applier/redo_test.go | 347 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ tests/integration_tests/_utils/check_sync_diff | 2 +- .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 ++++ .../integration_tests/force_replicate_table/run.sh | 4 +- .../open_protocol_handle_key_only/data/data.sql | 30 +- .../open_protocol_handle_key_only/run.sh | 2 + tests/integration_tests/run_group.sh | 5 +- 26 files changed, 974 insertions(+), 129 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh ๐Ÿงพ Pre-merged result: 7dd6241bd56139ff08ada1b98fc467877b87f31a 7dd6241bd f 41fc0de5f f 172d5e4f4 f โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/tiflow/rev-be15534-7dd6241) 203844608 bytes in 5.73 secs (35585670 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./dm/tests/download-integration-test-binaries.sh master % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 761 0 --:--:-- --:--:-- --:--:-- 773 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1662 0 --:--:-- --:--:-- --:--:-- 1708 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2505 0 --:--:-- --:--:-- --:--:-- 2562 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 608 0 --:--:-- --:--:-- --:--:-- 611 Download binaries... >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-04 15:27:26 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-04 15:27:38 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-04 15:28:00 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tidb-tools.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/d28ee6cd546e41862cf361e7a35552fb517b883e/centos7/tidb-tools.tar.gz 2024-05-04 15:28:17 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb-tools/d28ee6cd546e41862cf361e7a35552fb517b883e/centos7/tidb-tools.tar.gz [185439936/185439936] -> "tmp/tidb-tools.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-04 15:28:21 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download gh-ost-binary-linux-20200828140552.tar.gz from https://github.com/github/gh-ost/releases/download/v1.1.0/gh-ost-binary-linux-20200828140552.tar.gz 2024-05-04 15:28:24 URL:https://objects.githubusercontent.com/github-production-release-asset-2e65be/54378638/ac8d1400-e94a-11ea-9664-8b516d434420?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAVCODYLSA53PQK4ZA%2F20240504%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240504T072822Z&X-Amz-Expires=300&X-Amz-Signature=776d687a4c813e0f5e082880323e5ed3b8f7d5d4327cf119a73767eccb3fc381&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=54378638&response-content-disposition=attachment%3B%20filename%3Dgh-ost-binary-linux-20200828140552.tar.gz&response-content-type=application%2Foctet-stream [5141587/5141587] -> "tmp/gh-ost-binary-linux-20200828140552.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.2G drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:28 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:28 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 4 15:28 bin -rwxr-xr-x 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 regions-dump -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x 1 jenkins jenkins 127M Apr 8 13:50 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/gh-ost ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.2G drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:28 . drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:28 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 4 15:28 bin -rwxr-xr-x 1 jenkins jenkins 9.8M Aug 28 2020 gh-ost -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:10 regions-dump -rwxr-xr-x 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x 1 jenkins jenkins 127M Apr 8 13:50 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + [[ ! -f bin/dm-master.test ]] + echo 'Building binaries...' Building binaries... + make dm_integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading gorm.io/gorm v1.24.5 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading golang.org/x/crypto v0.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/docker/go-units v0.5.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/fatih/color v1.16.0 go: downloading golang.org/x/term v0.19.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/jinzhu/now v1.1.5 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/tidwall/pretty v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-72-g7dd6241bd" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 07:28:25" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7dd6241bd56139ff08ada1b98fc467877b87f31a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-72-g7dd6241bd"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dm-worker.test github.com/pingcap/tiflow/cmd/dm-worker \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-72-g7dd6241bd" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 07:28:25" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7dd6241bd56139ff08ada1b98fc467877b87f31a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-72-g7dd6241bd"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dm-master.test github.com/pingcap/tiflow/cmd/dm-master \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=1 GO111MODULE=on go test -p 3 -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-72-g7dd6241bd" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 07:28:25" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7dd6241bd56139ff08ada1b98fc467877b87f31a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-72-g7dd6241bd"' -c -cover -covermode=count \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dmctl.test github.com/pingcap/tiflow/cmd/dm-ctl \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-72-g7dd6241bd" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 07:28:25" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7dd6241bd56139ff08ada1b98fc467877b87f31a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-72-g7dd6241bd"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/dm/... \ -o bin/dm-syncer.test github.com/pingcap/tiflow/cmd/dm-syncer \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) ./dm/tests/prepare_tools.sh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + mkdir -p bin/dm-test-tools + cp -r ./dm/tests/bin/check_exit_safe_binlog ./dm/tests/bin/check_master_http_apis ./dm/tests/bin/check_master_online ./dm/tests/bin/check_master_online_http ./dm/tests/bin/check_worker_online ./bin/dm-test-tools + ls -alh ./bin total 1.1G drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:36 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:28 .. -rwxr-xr-x 1 jenkins jenkins 299M May 4 15:32 dm-master.test -rwxr-xr-x 1 jenkins jenkins 285M May 4 15:35 dm-syncer.test drwxr-sr-x 2 jenkins jenkins 4.0K May 4 15:36 dm-test-tools -rwxr-xr-x 1 jenkins jenkins 296M May 4 15:32 dm-worker.test -rwxr-xr-x 1 jenkins jenkins 215M May 4 15:35 dmctl.test + ls -alh ./bin/dm-test-tools total 344M drwxr-sr-x 2 jenkins jenkins 4.0K May 4 15:36 . drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:36 .. -rwxr-xr-x 1 jenkins jenkins 205M May 4 15:36 check_exit_safe_binlog -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_master_http_apis -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_master_online -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_master_online_http -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_worker_online + which ./bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test + which ./bin/dm-syncer.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test + which ./bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test + which ./bin/dmctl.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dmctl.test + which ./bin/dm-test-tools/check_master_online /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_master_online + which ./bin/dm-test-tools/check_worker_online /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-test-tools/check_worker_online [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/dm-integration-test/rev-be15534-7dd6241) 1506043904 bytes in 37.25 secs (40429818 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/gh-ost ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 2.2G drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:36 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:28 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 4 15:36 bin -rwxr-xr-x 1 jenkins jenkins 299M May 4 15:32 dm-master.test -rwxr-xr-x 1 jenkins jenkins 285M May 4 15:35 dm-syncer.test drwxr-sr-x 2 jenkins jenkins 4.0K May 4 15:36 dm-test-tools -rwxr-xr-x 1 jenkins jenkins 296M May 4 15:32 dm-worker.test -rwxr-xr-x 1 jenkins jenkins 215M May 4 15:35 dmctl.test -rwxr-xr-x 1 jenkins jenkins 9.8M May 4 15:36 gh-ost -rwxr-xr-x 1 jenkins jenkins 50M May 4 15:36 minio -rwxr-xr-x 1 jenkins jenkins 37M May 4 15:36 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 4 15:36 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 4 15:36 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 4 15:36 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 4 15:36 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 4 15:36 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 4 15:36 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 4 15:36 regions-dump -rwxr-xr-x 1 jenkins jenkins 32M May 4 15:36 stores-dump -rwxr-xr-x 1 jenkins jenkins 127M May 4 15:36 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 4 15:36 tidb-server -rwxr-xr-x 1 jenkins jenkins 418M May 4 15:36 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 4 15:36 xprog + ls -alh ./bin/dm-test-tools total 344M drwxr-sr-x 2 jenkins jenkins 4.0K May 4 15:36 . drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:36 .. -rwxr-xr-x 1 jenkins jenkins 205M May 4 15:36 check_exit_safe_binlog -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_master_http_apis -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_master_online -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_master_online_http -rwxr-xr-x 1 jenkins jenkins 35M May 4 15:36 check_worker_online [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 65.36 secs (50264077 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'TLS_GROUP') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. Created Pod: kubernetes jenkins-tiflow/dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520-mnbks-0vpfb [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-118b6b48-5e09-4ad2-a269-4274399996d7-djpwx-mr5x8 Created Pod: kubernetes jenkins-tiflow/dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2 Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520-mnbks-0vpfb is provisioned from template dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520-mnbks --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f564b43cfae6771cb65044ce56120750a505c1d1" jenkins/label: "dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520" name: "dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520-mnbks-0vpfb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520-mnbks-0vpfb" - name: "JENKINS_NAME" value: "dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520-mnbks-0vpfb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-6040ec40-d32b-46ef-aa05-ac203dc2e520-mnbks-0vpfb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@366390bf; decorates RemoteLauncher[hudson.remoting.Channel@678f6837:JNLP4-connect connection from 10.233.107.29/10.233.107.29:40446] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a-s8zv6-8ld06 Agent dm-it-118b6b48-5e09-4ad2-a269-4274399996d7-djpwx-mr5x8 is provisioned from template dm-it-118b6b48-5e09-4ad2-a269-4274399996d7-djpwx --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bccb76e29504c69c32d9eccf69c3ec9d706b24d9" jenkins/label: "dm-it-118b6b48-5e09-4ad2-a269-4274399996d7" name: "dm-it-118b6b48-5e09-4ad2-a269-4274399996d7-djpwx-mr5x8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-118b6b48-5e09-4ad2-a269-4274399996d7-djpwx-mr5x8" - name: "JENKINS_NAME" value: "dm-it-118b6b48-5e09-4ad2-a269-4274399996d7-djpwx-mr5x8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-118b6b48-5e09-4ad2-a269-4274399996d7-djpwx-mr5x8 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2-vsdx8-qdt5q [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@a921231; decorates RemoteLauncher[hudson.remoting.Channel@7233b53c:JNLP4-connect connection from 10.233.67.217/10.233.67.217:33334] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2 is provisioned from template dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "264c2165b1dd4cabb65132f4c783d6970de80f51" jenkins/label: "dm-it-4f1125b6-631d-42e7-8341-528c6efa3403" name: "dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2" - name: "JENKINS_NAME" value: "dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] checkout The recommended git tool is: git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Created Pod: kubernetes jenkins-tiflow/dm-it-62141bf0-8d60-41d4-947b-0c864f595686-2nxpm-zl8hp Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@340039ae; decorates RemoteLauncher[hudson.remoting.Channel@600350aa:JNLP4-connect connection from 10.233.66.198/10.233.66.198:49632] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 Agent dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p is provisioned from template dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "64bd00e62768eaba2b66795eed4611266726c850" jenkins/label: "dm-it-483c168e-4a6d-4447-8c64-19c04948818d" name: "dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p" - name: "JENKINS_NAME" value: "dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test [Pipeline] { Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4-v6bct-qscmf [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 4 07:38 . drwxr-sr-x 17 1000 1000 4.0K May 4 07:38 .. -rw------- 1 root 1000 1.7K May 4 07:38 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:38 ca.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:38 client-cert.pem -rw------- 1 root 1000 1.7K May 4 07:38 client-key.pem -rw------- 1 root 1000 1.7K May 4 07:38 private_key.pem -rw-r--r-- 1 root 1000 451 May 4 07:38 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:38 server-cert.pem -rw------- 1 root 1000 1.7K May 4 07:38 server-key.pem [Pipeline] } [Pipeline] node [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] { [Pipeline] cache > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1d29dbae; decorates RemoteLauncher[hudson.remoting.Channel@73e8e104:JNLP4-connect connection from 10.233.106.110/10.233.106.110:43448] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a-s8zv6-8ld06 is provisioned from template dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a-s8zv6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "305981c5342ed8eb8e517f09ee1aaca61bf92635" jenkins/label: "dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a" name: "dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a-s8zv6-8ld06" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a-s8zv6-8ld06" - name: "JENKINS_NAME" value: "dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a-s8zv6-8ld06" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-d54a5f37-0c83-4060-b4ae-65d0b833b08a-s8zv6-8ld06 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_dm_integration_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent dm-it-62141bf0-8d60-41d4-947b-0c864f595686-2nxpm-zl8hp is provisioned from template dm-it-62141bf0-8d60-41d4-947b-0c864f595686-2nxpm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1d0704d2af4c9d611b2e5ef15833e3bfb3b83a64" jenkins/label: "dm-it-62141bf0-8d60-41d4-947b-0c864f595686" name: "dm-it-62141bf0-8d60-41d4-947b-0c864f595686-2nxpm-zl8hp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-62141bf0-8d60-41d4-947b-0c864f595686-2nxpm-zl8hp" - name: "JENKINS_NAME" value: "dm-it-62141bf0-8d60-41d4-947b-0c864f595686-2nxpm-zl8hp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-62141bf0-8d60-41d4-947b-0c864f595686-2nxpm-zl8hp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Created Pod: kubernetes jenkins-tiflow/dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44r Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2 is provisioned from template dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "160ff243cee66ceda9cb3db4ae6f6abbbd897584" jenkins/label: "dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c" name: "dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2" - name: "JENKINS_NAME" value: "dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Agent dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2-vsdx8-qdt5q is provisioned from template dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2-vsdx8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "55fee3ceaf3fcd1771aac97ea737e26c93d7ed81" jenkins/label: "dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2" name: "dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2-vsdx8-qdt5q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2-vsdx8-qdt5q" - name: "JENKINS_NAME" value: "dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2-vsdx8-qdt5q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-aa0b6e80-b513-45ae-9f03-5d2401f889c2-vsdx8-qdt5q in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Agent dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4-v6bct-qscmf is provisioned from template dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4-v6bct --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "de62f5d7018531de38164dc7734069138541c2d3" jenkins/label: "dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4" name: "dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4-v6bct-qscmf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4-v6bct-qscmf" - name: "JENKINS_NAME" value: "dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4-v6bct-qscmf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-742b107f-b056-42e3-a92e-ed44b11dd6a4-v6bct-qscmf in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Still waiting to schedule task โ€˜dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44rโ€™ is offline Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 11.00 secs (298771818 bytes/sec) [Pipeline] { [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] echo [WARNING] label option is deprecated. To use a static pod template, use the 'inheritFrom' option. [Pipeline] podTemplate [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:38 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:38 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 4 15:38 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:38 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:38 .github -rw-r--r-- 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:38 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:38 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 4 15:38 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:38 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:38 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 4 15:38 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:38 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:38 engine -rwxr-xr-x 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:38 examples -rw-r--r-- 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:38 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 4 15:38 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:38 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:38 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:38 tests -rw-r--r-- 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:38 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] // timeout [Pipeline] sh Created Pod: kubernetes jenkins-tiflow/dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24-89ck2-ch0p9 [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4afc8f0a; decorates RemoteLauncher[hudson.remoting.Channel@16f4212b:JNLP4-connect connection from 10.233.86.226/10.233.86.226:50420] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + '[' TLS_GROUP == G00 ']' + echo 'run G00 test' run G00 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@43b250b9; decorates RemoteLauncher[hudson.remoting.Channel@25efd69e:JNLP4-connect connection from 10.233.108.123/10.233.108.123:49104] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4863d1fd; decorates RemoteLauncher[hudson.remoting.Channel@729535cf:JNLP4-connect connection from 10.233.127.65/10.233.127.65:49180] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@13e4575a; decorates RemoteLauncher[hudson.remoting.Channel@21d29134:JNLP4-connect connection from 10.233.71.187/10.233.71.187:38518] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7cf55e14; decorates RemoteLauncher[hudson.remoting.Channel@350bef12:JNLP4-connect connection from 10.233.69.223/10.233.69.223:39400] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] sh + make dm_integration_test_in_group GROUP=G00 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44r is provisioned from template dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "99e533e0a51e780f965e91cb0cec8671cfdc10c0" jenkins/label: "dm-it-153b7508-05f7-475f-a541-8c005c6b14f2" name: "dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44r" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44r" - name: "JENKINS_NAME" value: "dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44r" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44r in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 4 07:38 . drwxr-sr-x. 17 1000 1000 4.0K May 4 07:38 .. -rw-------. 1 root 1000 1.7K May 4 07:38 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:38 ca.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:38 client-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:38 client-key.pem -rw-------. 1 root 1000 1.7K May 4 07:38 private_key.pem -rw-r--r--. 1 root 1000 451 May 4 07:38 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:38 server-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:38 server-key.pem [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 4 07:38 . drwxr-sr-x. 17 1000 1000 4.0K May 4 07:38 .. -rw-------. 1 root 1000 1.7K May 4 07:38 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:38 ca.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:38 client-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:38 client-key.pem -rw-------. 1 root 1000 1.7K May 4 07:38 private_key.pem -rw-r--r--. 1 root 1000 451 May 4 07:38 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:38 server-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:38 server-key.pem [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] // container > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G00 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache Run cases: ha_cases_1 ha_cases_2 ha_cases2 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 4 07:38 . drwxr-sr-x 17 1000 1000 4.0K May 4 07:38 .. -rw------- 1 root 1000 1.7K May 4 07:38 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:38 ca.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:38 client-cert.pem -rw------- 1 root 1000 1.7K May 4 07:38 client-key.pem -rw------- 1 root 1000 1.7K May 4 07:38 private_key.pem -rw-r--r-- 1 root 1000 451 May 4 07:38 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:38 server-cert.pem -rw------- 1 root 1000 1.7K May 4 07:38 server-key.pem Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/run.sh... Verbose mode = false > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:38:47 CST 2024] <<<<<< start test_kill_master >>>>>> [Sat May 4 15:38:47 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master cluster [Sat May 4 15:38:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Sat May 4 15:38:49 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Sat May 4 15:38:49 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time Agent dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24-89ck2-ch0p9 is provisioned from template dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24-89ck2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5b29cac49ca6874d3bda5edcd5c3552d374950e0" jenkins/label: "dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24" name: "dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24-89ck2-ch0p9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24-89ck2-ch0p9" - name: "JENKINS_NAME" value: "dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24-89ck2-ch0p9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-80c80ae0-c97e-492b-adc4-d445fa995e24-89ck2-ch0p9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sat May 4 15:38:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" Created Pod: kubernetes jenkins-tiflow/dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0-7kxjb Created Pod: kubernetes jenkins-tiflow/dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql-6wf12 [Sat May 4 15:38:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase Still waiting to schedule task โ€˜dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql-6wf12โ€™ is offline Still waiting to schedule task โ€˜dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0-7kxjbโ€™ is offline Agent dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql-6wf12 is provisioned from template dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "42c4ee2a050825a4182d69dbcb778e1e3896757c" jenkins/label: "dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885" name: "dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql-6wf12" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql-6wf12" - name: "JENKINS_NAME" value: "dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql-6wf12" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0-7kxjb is provisioned from template dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" runUrl: "job/pingcap/job/tiflow/job/pull_dm_integration_test/1875/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "41aaa3dc523dc3177f51a81af9fc94fc6faa6b3b" jenkins/label: "dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe" name: "dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0-7kxjb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" image: "hub.pingcap.net/jenkins/mysql:5.7" name: "mysql1" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "--ssl=ON" - "--log-bin" - "--binlog-format=ROW" - "--enforce-gtid-consistency=ON" - "--gtid-mode=ON" - "--server-id=1" - "--default-authentication-plugin=mysql_native_password" env: - name: "MYSQL_ROOT_PASSWORD" value: "123456" - name: "MYSQL_TCP_PORT" value: "3307" image: "registry-mirror.pingcap.net/library/mysql:8.0.21" name: "mysql2" resources: limits: memory: "4Gi" cpu: "2" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0-7kxjb" - name: "JENKINS_NAME" value: "dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0-7kxjb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on dm-it-579f3e53-1aa2-46d9-948e-8d6e3a24a885-tkkql-6wf12 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Running on dm-it-9d01bbc4-b94b-4693-a9a9-636e814328fe-cslg0-7kxjb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 10.85 secs (302806242 bytes/sec) [Pipeline] { [Pipeline] cache use sync_diff_inspector to check increment data check diff successfully [Sat May 4 15:39:00 CST 2024] <<<<<< finish test_running >>>>>> kill dm-master1 wait process dm-master1 exit... wait process dm-master1 exit... process dm-master1 already exit waiting 5 seconds check task is running HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive check master2,3 are running dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 use sync_diff_inspector to check increment2 data now! check diff successfully [Sat May 4 15:39:09 CST 2024] <<<<<< finish test_kill_master >>>>>> [Sat May 4 15:39:09 CST 2024] <<<<<< start test_kill_and_isolate_worker >>>>>> [Sat May 4 15:39:09 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 19.62 secs (167458752 bytes/sec) [Pipeline] { [Pipeline] checkout [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { The recommended git tool is: git [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh [Pipeline] sh wait process dm-master.test exit... + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:38 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:39 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 4 15:38 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:38 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:38 .github -rw-r--r--. 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:38 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:38 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 15:38 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:38 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:38 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 4 15:38 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:38 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:38 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:38 examples -rw-r--r--. 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:38 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 4 15:38 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:38 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:38 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:38 tests -rw-r--r--. 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:38 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2d222c6b; decorates RemoteLauncher[hudson.remoting.Channel@62be4852:JNLP4-connect connection from 10.233.105.66/10.233.105.66:49094] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:39 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:39 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 4 15:39 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:39 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:39 .github -rw-r--r--. 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:39 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:39 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 15:39 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:39 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:39 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 4 15:39 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:39 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:39 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:39 examples -rw-r--r--. 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:39 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 4 15:39 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:39 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:39 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:39 tests -rw-r--r--. 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:39 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] } Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] } wait process dm-master.test exit... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } [Pipeline] cache wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" wait process dm-master.test exit... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 17.12 secs (191957001 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] sh The recommended git tool is: git wait process dm-master.test exit... [Pipeline] // timeout [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:39 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:39 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 4 15:39 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:39 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:39 .github -rw-r--r-- 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:39 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:39 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 4 15:39 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:39 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:39 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 4 15:39 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:39 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:39 engine -rwxr-xr-x 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:39 examples -rw-r--r-- 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:39 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 4 15:39 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:39 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:39 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:39 tests -rw-r--r-- 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:39 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] // timeout + '[' TLS_GROUP == G01 ']' + echo 'run G01 test' run G01 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@565f2515; decorates RemoteLauncher[hudson.remoting.Channel@490cb71b:JNLP4-connect connection from 10.233.72.70/10.233.72.70:33162] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3253e9f; decorates RemoteLauncher[hudson.remoting.Channel@6d416956:JNLP4-connect connection from 10.233.73.37/10.233.73.37:52722] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@28888338; decorates RemoteLauncher[hudson.remoting.Channel@43fb7933:JNLP4-connect connection from 10.233.67.144/10.233.67.144:60920] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + make dm_integration_test_in_group GROUP=G01 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { wait process dm-master.test exit... [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + '[' TLS_GROUP == G02 ']' + echo 'run G02 test' run G02 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G02 [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] stage [Pipeline] { (Test) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { wait process dm-master.test exit... [Pipeline] // timeout [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G01 Run cases: ha_cases3 ha_cases3_1 ha_master tracker_ignored_ddl validator_checkpoint incompatible_ddl_changes ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + '[' TLS_GROUP == G04 ']' + echo 'run G04 test' run G04 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G02 Run cases: handle_error handle_error_2 handle_error_3 adjust_gtid async_checkpoint_flush binlog_parse case_sensitive checkpoint_transaction check_task ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + make dm_integration_test_in_group GROUP=G04 [Pipeline] sh wait process dm-master.test exit... /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 4 07:39 . drwxr-sr-x. 17 1000 1000 4.0K May 4 07:39 .. -rw-------. 1 root 1000 1.7K May 4 07:39 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:39 ca.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:39 client-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:39 client-key.pem -rw-------. 1 root 1000 1.7K May 4 07:39 private_key.pem -rw-r--r--. 1 root 1000 451 May 4 07:39 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:39 server-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:39 server-key.pem > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Pipeline] sh Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 4 07:39 . drwxr-sr-x 17 1000 1000 4.0K May 4 07:39 .. -rw------- 1 root 1000 1.7K May 4 07:39 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 ca.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 client-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 client-key.pem -rw------- 1 root 1000 1.7K May 4 07:39 private_key.pem -rw-r--r-- 1 root 1000 451 May 4 07:39 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 server-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 server-key.pem Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] sh Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 4 07:39 . drwxr-sr-x. 17 1000 1000 4.0K May 4 07:39 .. -rw-------. 1 root 1000 1.7K May 4 07:39 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:39 ca.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:39 client-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:39 client-key.pem -rw-------. 1 root 1000 1.7K May 4 07:39 private_key.pem -rw-r--r--. 1 root 1000 451 May 4 07:39 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:39 server-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:39 server-key.pem [Pipeline] container [Pipeline] { wait process dm-master.test exit... [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G04 Run cases: import_goroutine_leak incremental_mode initial_unit drop_column_with_index duplicate_event expression_filter extend_column fake_rotate_event import_goroutine_leak not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 4 07:39 . drwxr-sr-x 17 1000 1000 4.0K May 4 07:39 .. -rw------- 1 root 1000 1.7K May 4 07:39 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 ca.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 client-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 client-key.pem -rw------- 1 root 1000 1.7K May 4 07:39 private_key.pem -rw-r--r-- 1 root 1000 451 May 4 07:39 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 server-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 server-key.pem [Pipeline] { [Pipeline] container [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] sh > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 4 07:39 . drwxr-sr-x 17 1000 1000 4.0K May 4 07:39 .. -rw------- 1 root 1000 1.7K May 4 07:39 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 ca.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 client-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 client-key.pem -rw------- 1 root 1000 1.7K May 4 07:39 private_key.pem -rw-r--r-- 1 root 1000 451 May 4 07:39 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 server-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 server-key.pem VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [ha_cases3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:39:47 CST 2024] <<<<<< start test_stop_task >>>>>> [Sat May 4 15:39:47 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table wait process dm-master.test exit... [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x 2 root 1000 4.0K May 4 07:39 . drwxr-sr-x 17 1000 1000 4.0K May 4 07:39 .. -rw------- 1 root 1000 1.7K May 4 07:39 ca-key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 ca.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 client-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 client-key.pem -rw------- 1 root 1000 1.7K May 4 07:39 private_key.pem -rw-r--r-- 1 root 1000 451 May 4 07:39 public_key.pem -rw-r--r-- 1 root 1000 1.1K May 4 07:39 server-cert.pem -rw------- 1 root 1000 1.7K May 4 07:39 server-key.pem [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] } [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [handle_error] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:39:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 wait process dm-master.test exit... start running case: [incremental_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:39:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time import prepare data start DM worker and master [Sat May 4 15:39:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master1.toml >>>>>> [Sat May 4 15:39:49 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master2.toml >>>>>> [Sat May 4 15:39:49 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:39:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:39:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source1.yaml" [Sat May 4 15:39:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:39:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error/source2.yaml" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8263 is alive rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Sat May 4 15:39:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sat May 4 15:39:53 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source1.yaml" [Sat May 4 15:39:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb2 add column c int default 100; alter table handle_error.tb2 add primary key (c);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog inject test -s mysql-replica-01 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique(c);" dmctl test cmd: "binlog inject test -s mysql-replica-02 alter table handle_error.tb2 drop primary key; alter table handle_error.tb2 add unique(c);" run tidb sql failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3/source2.yaml" dmctl test cmd: "list-member" [Sat May 4 15:39:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... [Sat May 4 15:39:57 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:39:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8264 is alive [Sat May 4 15:39:58 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time dmctl test cmd: "stop-task test" [Sat May 4 15:39:58 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR >>>>>> [Sat May 4 15:39:58 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" rpc addr 127.0.0.1:18262 is alive [Sat May 4 15:39:59 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test "ErrScope": "downstream" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... dmctl test cmd: "start-relay -s mysql-replica-02 worker1" rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml " wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-01 worker2" dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test2" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" got=2 expected=2 wait process dm-master.test exit... wait process dm-master.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 13.12 secs (250507949 bytes/sec) [Pipeline] { [Pipeline] cache wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... use sync_diff_inspector to check increment data check diff successfully check diff successfully [Sat May 4 15:40:07 CST 2024] <<<<<< finish test_multi_task_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "list-member" [Sat May 4 15:40:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:40:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive got=2 expected=2 start dumping SQLs into source stop tasks test dmctl test cmd: "stop-task test" wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "query-status test" got=1 expected=1 stop tasks test2 dmctl test cmd: "stop-task test2" got=3 expected=3 dmctl test cmd: "query-status test2" got=1 expected=1 wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "list-member --worker" got=2 expected=2 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task.yaml" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-worker.test exit... process dm-worker.test already exit dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source update /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "stop-relay -s mysql-replica-02 worker1" clean source table got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" got=2 expected=2 dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source stop /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/incremental_mode/source2.yaml" dmctl test cmd: "start-relay -s mysql-replica-02 worker1" got=2 expected=2 dmctl test cmd: "start-relay -s mysql-replica-01 worker2" got=2 expected=2 start task in incremental mode with zero gtid/pos [Sat May 4 15:40:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=4 expected=4 start tasks /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3/conf/dm-task2.yaml" import prepare data start DM worker and master cluster [Sat May 4 15:40:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master1.toml >>>>>> [Sat May 4 15:40:14 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master2.toml >>>>>> [Sat May 4 15:40:14 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:40:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test2" got=4 expected=4 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 9.03 secs (363867695 bytes/sec) [Pipeline] { [Pipeline] cache wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sat May 4 15:40:17 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source1.yaml" wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:40:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sat May 4 15:40:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:40:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_1/source2.yaml" rpc addr 127.0.0.1:8263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase [2024/05/04 15:40:15.410 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=4b31d77d-09e9-11ef-ba76-f2ccbc6fedf7.000001/dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p-bin.000001] [2024/05/04 15:40:15.427 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=4b31d77d-09e9-11ef-ba76-f2ccbc6fedf7.000001/dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p-bin.000001] [2024/05/04 15:40:19.836 +08:00] [INFO] [relay.go:96] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=4b31d77d-09e9-11ef-ba76-f2ccbc6fedf7.000001/dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p-bin.000001] [2024/05/04 15:40:19.853 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=4b31d77d-09e9-11ef-ba76-f2ccbc6fedf7.000001/dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p-bin.000001] [2024/05/04 15:40:19.876 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=4b31d77d-09e9-11ef-ba76-f2ccbc6fedf7.000001/dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p-bin.000001] [2024/05/04 15:40:19.900 +08:00] [INFO] [relay.go:125] ["current earliest active relay log"] [task=test] [unit="binlog replication"] ["active relay log"=4b31d77d-09e9-11ef-ba76-f2ccbc6fedf7.000001/dm-it-483c168e-4a6d-4447-8c64-19c04948818d-tvd4v-96r1p-bin.000001] dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "pause-task test" dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Sat May 4 15:40:25 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE pessimistic >>>>>> [Sat May 4 15:40:25 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int default 100; alter table handle_error.tb1 add primary key (c);" check log contain failed 1-th time, retry later use sync_diff_inspector to check increment data check diff successfully [Sat May 4 15:40:26 CST 2024] <<<<<< finish test_running >>>>>> dmctl test cmd: "start-relay -s mysql-replica-02 worker2" got=2 expected=2 kill dm-worker2 wait process dm-worker2 exit... dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Sat May 4 15:40:29 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE optimistic >>>>>> [Sat May 4 15:40:29 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker2 exit... process dm-worker2 already exit dmctl test cmd: "query-status test" got=1 expected=1 [Sat May 4 15:40:30 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01,mysql-replica-02 alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8264 is alive wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Sat May 4 15:40:31 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time got=4 expected=4 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker3 worker4" got=3 expected=3 restart dm-worker3 dmctl test cmd: "stop-task test" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 15.18 secs (216478872 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] } [Pipeline] sh [Sat May 4 15:40:33 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 pessimistic >>>>>> [Sat May 4 15:40:33 CST 2024] <<<<<< start DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:40 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:40 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 4 15:39 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:39 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:39 .github -rw-r--r--. 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:40 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:40 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 15:40 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:40 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 4 15:40 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:40 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:40 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 examples -rw-r--r--. 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 4 15:40 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:40 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:40 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:40 tests -rw-r--r--. 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:40 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:40 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 4 15:40 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:40 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 .github -rw-r--r-- 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:40 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:40 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 4 15:40 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:40 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 4 15:40 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:40 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:40 engine -rwxr-xr-x 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 examples -rw-r--r-- 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 4 15:40 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:40 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:40 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:40 tests -rw-r--r-- 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache wait process dm-worker3 exit... + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:40 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:40 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 4 15:40 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:40 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 .github -rw-r--r--. 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:40 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:40 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 15:40 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:40 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 4 15:40 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:40 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:40 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 examples -rw-r--r--. 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 4 15:40 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:40 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:40 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:40 tests -rw-r--r--. 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20); alter table handle_error.tb1 add primary key (c);" dmctl test cmd: "query-status test" wait process dm-worker3 exit... process dm-worker3 already exit wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Sat May 4 15:40:35 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb2 add column c varchar(20); alter table handle_error.tb2 add primary key (c);" check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8264 is alive isolate dm-worker4 kill dm-worker4 dmctl test cmd: "stop-task test" wait process dm-worker4 exit... wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" [Sat May 4 15:40:38 CST 2024] <<<<<< finish DM-INJECT_DDL_ERROR_SHARDING_BASE2 optimistic >>>>>> [Sat May 4 15:40:38 CST 2024] <<<<<< start DM-LIST_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker4 exit... process dm-worker4 already exit [Sat May 4 15:40:38 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=3 expected=3 isolate dm-worker3 kill dm-worker3 wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:40:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker3 exit... dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Unsupported modify column: this column has primary key flag count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:40:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/incremental_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker3 exit... process dm-worker3 already exit [Sat May 4 15:40:40 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker3.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 disable isolate dm-worker4 kill dm-worker4 rpc addr 127.0.0.1:8263 is alive wait process dm-worker4 exit... got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:14435" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog inject test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:14627 alter table handle_error.tb1 drop primary key; alter table handle_error.tb1 add unique (c);" wait process dm-worker4 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog list test" dmctl test cmd: "binlog skip test" process dm-worker4 already exit [Sat May 4 15:40:43 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=3 command: query-status test "stage": "Running" count: 1 != expected: 3, failed the 0-th time, will retry again run tidb sql failed 1-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=3 expected=3 query-status from all dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" [Sat May 4 15:40:45 CST 2024] <<<<<< finish DM-LIST_ERROR >>>>>> [Sat May 4 15:40:45 CST 2024] <<<<<< start DM-SKIP_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" got=3 expected=3 dmctl test cmd: "pause-task test" restart worker4 got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-worker4 exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" got=3 expected=3 check diff successfully wait process dm-worker4 exit... process dm-worker4 already exit [Sat May 4 15:40:48 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time [Sat May 4 15:40:48 CST 2024] <<<<<< finish DM-SKIP_ERROR >>>>>> [Sat May 4 15:40:48 CST 2024] <<<<<< start DM-REPLACE_ERROR >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:18262 is alive dmctl test cmd: "resume-task test" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 10.29 secs (319337190 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 50 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column new_col text, add column c int; alter table handle_error.tb1 add unique(c);" [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] cache run tidb sql failed 1-th time, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 3648 0 3648 0 0 483k 0 --:--:-- --:--:-- --:--:-- 508k dmctl test cmd: "stop-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "stop-relay -s mysql-replica-02 worker1" got=2 expected=2 dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" got=2 expected=2 dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "check-task /tmp/dm_test/incremental_mode/dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/incremental_mode/dm-task.yaml --remove-meta" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive use sync_diff_inspector to check increment2 data now! check diff successfully [Sat May 4 15:40:51 CST 2024] <<<<<< finish test_kill_and_isolate_worker >>>>>> 3 dm-master alive 3 dm-worker alive 0 dm-syncer alive + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:40 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:40 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 4 15:40 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:40 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 .github -rw-r--r-- 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:40 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:40 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 4 15:40 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:40 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 4 15:40 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:40 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:40 engine -rwxr-xr-x 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 examples -rw-r--r-- 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 4 15:40 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:40 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:40 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:40 tests -rw-r--r-- 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Sat May 4 15:40:52 CST 2024] <<<<<< finish DM-REPLACE_ERROR >>>>>> [Sat May 4 15:40:52 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "stop-task test" [Sat May 4 15:40:56 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING pessimistic >>>>>> [Sat May 4 15:40:56 CST 2024] <<<<<< start DM-CROSS_DDL_SHARDING optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:40:56 CST 2024] <<<<<< test case incremental_mode success! >>>>>> start running case: [initial_unit] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... failpoint=github.com/pingcap/tiflow/dm/syncer/LoadCheckpointFailed=return(1152) [Sat May 4 15:40:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=4 expected=4 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:40:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sat May 4 15:40:58 CST 2024] <<<<<< finish DM-CROSS_DDL_SHARDING optimistic >>>>>> [Sat May 4 15:40:58 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/0/source1.yaml" start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit got=4 expected=4 run tidb sql failed 1-th time, retry later process dm-syncer.test already exit [Sat May 4 15:41:00 CST 2024] <<<<<< test case ha_cases_1 success! >>>>>> start running case: [ha_cases_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:41:00 CST 2024] <<<<<< start test_kill_master_in_sync >>>>>> [Sat May 4 15:41:00 CST 2024] <<<<<< start test_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "stop-task test" [Sat May 4 15:41:02 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE pessimistic >>>>>> [Sat May 4 15:41:02 CST 2024] <<<<<< start DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" resume task will also initial failed dmctl test cmd: "resume-task test" import prepare data start DM worker and master cluster [Sat May 4 15:41:02 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Sat May 4 15:41:02 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Sat May 4 15:41:02 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column a varchar(20); alter table handle_error.tb1 add unique idx(a); alter table handle_error.tb1 add column b varchar(20); alter table handle_error.tb1 add unique idx2(b);"" got=3 expected=3 dmctl test cmd: "query-status test" got=4 expected=4 run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Sat May 4 15:41:06 CST 2024] <<<<<< finish DM-REPLACE_ERROR_MULTIPLE optimistic >>>>>> [Sat May 4 15:41:06 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sat May 4 15:41:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:41:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:41:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Sat May 4 15:41:10 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP pessimistic >>>>>> [Sat May 4 15:41:10 CST 2024] <<<<<< start DM-EXEC_ERROR_SKIP optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test Error 1062 (23000): Duplicate count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "stop-task test" [Sat May 4 15:41:14 CST 2024] <<<<<< finish DM-EXEC_ERROR_SKIP optimistic >>>>>> [Sat May 4 15:41:14 CST 2024] <<<<<< start DM-SKIP_INCOMPATIBLE_DDL >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=2 expected=2 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/run.sh: line 556: hello: command not found dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 dmctl test cmd: "binlog skip test" check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:41:17 CST 2024] <<<<<< finish DM-SKIP_INCOMPATIBLE_DDL >>>>>> [Sat May 4 15:41:17 CST 2024] <<<<<< start DM-4202 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" use sync_diff_inspector to check increment data check diff successfully [Sat May 4 15:41:16 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source wait process dm-master1 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" check diff successfully dmctl test cmd: "resume-task test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:35724" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-master1 exit... process dm-master1 already exit wait and check task running wait process dm-master.test exit... HTTP 127.0.0.1:8361/apis/v1alpha1/status/test is alive dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit [Sat May 4 15:41:21 CST 2024] <<<<<< finish DM-4202 >>>>>> wait process dm-worker.test exit... [Sat May 4 15:41:22 CST 2024] <<<<<< start DM-4204 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-worker.test exit... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 30.81 secs (106622477 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $DM_CODECOV_TOKEN or $DM_COVERALLS_TOKEN dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit got=2 expected=2 dmctl test cmd: "stop-task test" [Sat May 4 15:41:24 CST 2024] <<<<<< finish DM-4204 >>>>>> + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:41 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:41 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 4 15:40 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:40 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:40 .github -rw-r--r-- 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:41 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:41 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 4 15:41 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:41 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 4 15:41 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:41 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:41 engine -rwxr-xr-x 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 examples -rw-r--r-- 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 4 15:41 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:41 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:41 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:41 tests -rw-r--r-- 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break failpoint=github.com/pingcap/tiflow/dm/syncer/dbconn/GetMasterStatusFailed=return(1152) [Sat May 4 15:41:24 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sat May 4 15:41:25 CST 2024] <<<<<< start DM-4206 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] // container [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] cache dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:39469 -s mysql-replica-01" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:41:26 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:39276 -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:36408 -s mysql-replica-02" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:36627 -s mysql-replica-02" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/initial_unit/1/source1.yaml" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" start task and query status, the sync unit will initial failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" [Sat May 4 15:41:28 CST 2024] <<<<<< finish DM-4206 pessimistic >>>>>> [Sat May 4 15:41:28 CST 2024] <<<<<< start DM-4206 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:41438 -s mysql-replica-01" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:41245 -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:38566 -s mysql-replica-02" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:38785 -s mysql-replica-02" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" resume task will also initial failed dmctl test cmd: "resume-task test" [Sat May 4 15:41:31 CST 2024] <<<<<< finish DM-4206 optimistic >>>>>> [Sat May 4 15:41:32 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" reset go failpoints, and need restart dm-worker then resume task, task will recover success got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:43214" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:40724" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-worker.test exit... [Sat May 4 15:41:35 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Sat May 4 15:41:35 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:41:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:44990" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:42663" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later dmctl test cmd: "stop-task test" [Sat May 4 15:41:39 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> [Sat May 4 15:41:39 CST 2024] <<<<<< start DM-4207 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:47024" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:44602" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Sat May 4 15:41:42 CST 2024] <<<<<< finish DM-4207 pessimistic >>>>>> [Sat May 4 15:41:42 CST 2024] <<<<<< start DM-4207 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 14.24 secs (230778715 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] sh dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:49058" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:41 . drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:41 .. -rw-r--r-- 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx 1 jenkins jenkins 10 May 4 15:41 .dockerignore -> .gitignore -rw-r--r-- 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:41 .git drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 .github -rw-r--r-- 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r-- 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r-- 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r-- 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r-- 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r-- 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r-- 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r-- 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r-- 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r-- 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r-- 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r-- 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:41 bin drwxr-sr-x 19 jenkins jenkins 4.0K May 4 15:41 cdc drwxr-sr-x 6 jenkins jenkins 4.0K May 4 15:41 cdcv2 drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:41 cmd drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 deployments drwxr-sr-x 26 jenkins jenkins 4.0K May 4 15:41 dm drwxr-sr-x 7 jenkins jenkins 4.0K May 4 15:41 docs drwxr-sr-x 13 jenkins jenkins 4.0K May 4 15:41 engine -rwxr-xr-x 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 examples -rw-r--r-- 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r-- 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 metrics drwxr-sr-x 44 jenkins jenkins 4.0K May 4 15:41 pkg drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:41 proto drwxr-sr-x 3 jenkins jenkins 4.0K May 4 15:41 scripts drwxr-sr-x 5 jenkins jenkins 4.0K May 4 15:41 tests -rw-r--r-- 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x 4 jenkins jenkins 4.0K May 4 15:41 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:46541" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Sat May 4 15:41:44 CST 2024] <<<<<< finish DM-4207 optimistic >>>>>> [Pipeline] } [Sat May 4 15:41:45 CST 2024] <<<<<< start DM-4209 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" run tidb sql failed 1-th time, retry later [Pipeline] } [Pipeline] // timeout [Pipeline] // timeout [Pipeline] sh [Pipeline] sh + '[' TLS_GROUP == G05 ']' + echo 'run G05 test' run G05 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ + make dm_integration_test_in_group GROUP=G05 + '[' TLS_GROUP == G07 ']' + echo 'run G07 test' run G07 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + make dm_integration_test_in_group GROUP=G07 start task after restarted dm-worker, the sync unit will initial success dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/initial_unit/conf/dm-task.yaml" check diff successfully dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt wait process dm-master.test exit... process dm-master.test already exit [Pipeline] // timeout dmctl test cmd: "stop-task test" [Pipeline] sh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G05 Run cases: load_interrupt many_tables online_ddl foreign_key full_mode gbk gtid ha_cases http_proxies load_interrupt not exist ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 15:41:49 CST 2024] <<<<<< finish DM-4209 pessimistic >>>>>> [Sat May 4 15:41:49 CST 2024] <<<<<< start DM-4209 optimistic >>>>>> [Pipeline] } [Pipeline] // timeout /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G07 Run cases: shardddl1 shardddl1_1 shardddl2 shardddl2_1 ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/double-source-optimistic.yaml --remove-meta" + '[' TLS_GROUP == G09 ']' + echo 'run G09 test' run G09 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + make dm_integration_test_in_group GROUP=G09 [Pipeline] // timeout [Pipeline] sh + '[' TLS_GROUP == G06 ']' + echo 'run G06 test' run G06 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] sh + make dm_integration_test_in_group GROUP=G06 use sync_diff_inspector to check increment data check diff successfully check diff successfully [Sat May 4 15:41:37 CST 2024] <<<<<< finish test_stop_task >>>>>> 3 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:41:46 CST 2024] <<<<<< test case ha_cases3 success! >>>>>> start running case: [ha_cases3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:41:46 CST 2024] <<<<<< start test_isolate_master_and_worker >>>>>> [Sat May 4 15:41:46 CST 2024] <<<<<< start test_multi_task_running >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table import prepare data start DM worker and master [Sat May 4 15:41:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master1.toml >>>>>> [Sat May 4 15:41:49 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> [Sat May 4 15:41:49 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time + '[' TLS_GROUP == G08 ']' + echo 'run G08 test' run G08 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt [Pipeline] sh + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 4 07:41 . drwxr-sr-x. 17 1000 1000 4.0K May 4 07:41 .. -rw-------. 1 root 1000 1.7K May 4 07:41 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 ca.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 client-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:41 client-key.pem -rw-------. 1 root 1000 1.7K May 4 07:41 private_key.pem -rw-r--r--. 1 root 1000 451 May 4 07:41 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 server-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:41 server-key.pem [Pipeline] sh dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01" + make dm_integration_test_in_group GROUP=G08 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G09 Run cases: import_v10x sharding2 ha new_collation_off only_dml openapi s3_dumpling_lightning sequence_sharding_optimistic ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test -s mysql-replica-02" + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 4 07:41 . drwxr-sr-x. 17 1000 1000 4.0K May 4 07:41 .. -rw-------. 1 root 1000 1.7K May 4 07:41 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 ca.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 client-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:41 client-key.pem -rw-------. 1 root 1000 1.7K May 4 07:41 private_key.pem -rw-r--r--. 1 root 1000 451 May 4 07:41 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 server-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:41 server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt rpc addr 127.0.0.1:8261 is alive VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. run tidb sql failed 1-th time, retry later + mkdir /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl + cp -r /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ + ls -alh /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ total 40K drwxr-sr-x. 2 root 1000 4.0K May 4 07:41 . drwxr-sr-x. 17 1000 1000 4.0K May 4 07:41 .. -rw-------. 1 root 1000 1.7K May 4 07:41 ca-key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 ca.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 client-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:41 client-key.pem -rw-------. 1 root 1000 1.7K May 4 07:41 private_key.pem -rw-r--r--. 1 root 1000 451 May 4 07:41 public_key.pem -rw-r--r--. 1 root 1000 1.1K May 4 07:41 server-cert.pem -rw-------. 1 root 1000 1.7K May 4 07:41 server-key.pem [Pipeline] // timeout [Pipeline] sh rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive operate mysql config to worker [Sat May 4 15:41:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time version() 5.7.26-log version() 8.0.21 start running case: [shardddl1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:41:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [many_tables] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G08 + '[' TLS_GROUP == G03 ']' + echo 'run G03 test' run G03 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ Run cases: shardddl3 shardddl3_1 shardddl4 shardddl4_1 sharding sequence_sharding sequence_sharding_removemeta ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process dm-worker.test exit... wait process tidb-server exit... process tidb-server already exit Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] } [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G06 rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source1.yaml" + make dm_integration_test_in_group GROUP=G03 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt Run cases: relay_interrupt safe_mode sequence_safe_mode lightning_load_task lightning_mode metrics ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] // container [Pipeline] // container wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:41:53 CST 2024] <<<<<< test case initial_unit success! >>>>>> start running case: [drop_column_with_index] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/run.sh... Verbose mode = false [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:41:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Pipeline] cache 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:41:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time version() 5.7.26-log version() 8.0.21 start running case: [import_v10x] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit source1 master status, dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2-bin.000001, 1049, 4de86fa8-09e9-11ef-981f-26861879658f:1-4 dmctl test cmd: "stop-task test" source2 master status, dm-it-a6f31034-24de-4563-91e7-208be4ff2e9c-2sf4q-33wq2-bin.000001, 1158, 4e520995-09e9-11ef-bc68-26861879658f:1-4 [Sat May 4 15:41:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G03 Run cases: dmctl_advance dmctl_basic dmctl_command downstream_diff_index downstream_more_column ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 15:41:54 CST 2024] <<<<<< finish DM-4209 optimistic >>>>>> [Sat May 4 15:41:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source1.yaml" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:41:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-worker1.toml >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. [Sat May 4 15:41:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8262 alive the 1-th time version() 5.7.26-log version() 8.0.21 start running case: [shardddl3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:41:54 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:41:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [relay_interrupt] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetGlobalVariableFailed=return("server_uuid,1152") rpc addr 127.0.0.1:8263 is alive VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. [Sat May 4 15:41:55 CST 2024] <<<<<< start DM-4211 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases3_1/source2.yaml" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1/source2.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/drop_column_with_index/source1.yaml" [Sat May 4 15:41:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/drop_column_with_index/conf/dm-task.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:41:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/import_v10x/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [dmctl_advance] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm [Sat May 4 15:41:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_advance/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test --binlog-pos dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:54386 alter table handle_error.tb1 add column c int;" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" [Sat May 4 15:41:56 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:41:56 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" got=3 expected=3 got=1 expected=1 got=1 expected=1 dmctl test cmd: "query-status test" start prepare_data wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "query-status test" [Sat May 4 15:41:57 CST 2024] <<<<<< finish DM-4211 >>>>>> got=3 expected=3 dmctl test cmd: "config task test --path /tmp/dm_test/import_v10x/task.yaml" got=1 expected=1 rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "shard-ddl-lock unlock" dmctl test cmd: "shard-ddl-lock unlock lock_id --force-remove=invalid-force-remove" dmctl test cmd: "handle-error" got=1 expected=1 got=1 expected=1 got=1 expected=1 dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" dmctl test cmd: "handle-error test-task --binlog-pos mysql-bin:shoud-bin-digital skip" dmctl test cmd: "handle-error test-task replace alter table tb add column a int; alter table tb2 b int;" got=1 expected=1 got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8264 is alive [Sat May 4 15:41:57 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker4.toml >>>>>> wait for rpc addr 127.0.0.1:18262 alive the 1-th time dmctl test cmd: "handle-error test-task wrong_operation" dmctl test cmd: "binlog" wait process dm-worker exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:41:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "binlog skip test-task --binlog-pos mysql-bin:should-be-digital" dmctl test cmd: "binlog replace test-task alter table tb add column a int; alter table tb2 b int;" [Sat May 4 15:41:58 CST 2024] <<<<<< start DM-4213 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source1.yaml" [Sat May 4 15:41:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "binlog inject test-task alter table tb add column a int; alter table tb2 b int;" dmctl test cmd: "binlog list test-task --binlog-pos mysql-bin:should-be-digital" dmctl test cmd: "binlog wrong_operation test-task" dmctl test cmd: "binlog-schema" rpc addr 127.0.0.1:18262 is alive [Sat May 4 15:41:58 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker5.toml >>>>>> wait for rpc addr 127.0.0.1:18263 alive the 1-th time dmctl test cmd: "binlog-schema test" dmctl test cmd: "binlog-schema list test" wait process dm-worker exit... process dm-worker already exit [Sat May 4 15:41:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sat May 4 15:41:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/0/source1.yaml" dmctl test cmd: "binlog-schema update test" dmctl test cmd: "binlog-schema update test test_db test_table --from-source" dmctl test cmd: "binlog-schema update test test_db test_table --from-source --from-target" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3/source2.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "binlog-schema update test test_db test_table schema.sql --from-source" dmctl test cmd: "binlog-schema delete test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sat May 4 15:41:59 CST 2024] <<<<<< start DM-COMPACT >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" check diff successfully 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:18263 is alive start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task2.yaml " dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-task.yaml " [Sat May 4 15:42:00 CST 2024] <<<<<< finish DM-4213 >>>>>> [Sat May 4 15:42:00 CST 2024] <<<<<< start DM-071 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" waiting for asynchronous relay and subtask to be started wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=4 expected=4 got=2 expected=2 dmctl test cmd: "query-status test2" got=4 expected=4 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully check diff successfully flush logs to force rotate binlog file 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive apply increment data before restart dm-worker to ensure entering increment phase wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... [Sat May 4 15:42:02 CST 2024] <<<<<< finish DM-071 pessimistic >>>>>> dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-worker.test exit... dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status" dmctl test cmd: "resume-task test" check diff successfully dmctl test cmd: "query-status test" got=2 expected=2 got=0 expected=1 command: query-status test "synced": true count: 0 != expected: 1, failed the 0-th time, will retry again reset go failpoints, and need restart dm-worker then resume task, task will recover success wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:03 CST 2024] <<<<<< test case dmctl_advance success! >>>>>> start running case: [dmctl_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm dmctl test cmd: "any command" wait process dm-master.test exit... process dm-master.test already exit [Sat May 4 15:42:03 CST 2024] <<<<<< start DM-073 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:04 CST 2024] <<<<<< test case import_v10x success! >>>>>> start running case: [sharding2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... got=2 expected=2 got=1 expected=1 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:42:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:42:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> [Sat May 4 15:42:05 CST 2024] <<<<<< finish DM-073 pessimistic >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:06 CST 2024] <<<<<< test case handle_error success! >>>>>> start running case: [handle_error_2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/run.sh... Verbose mode = false wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 1-th time, retry later dmctl test cmd: "any command" [Sat May 4 15:42:06 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source1.yaml" use sync_diff_inspector to check increment data check diff successfully check diff successfully [Sat May 4 15:42:07 CST 2024] <<<<<< finish test_multi_task_running >>>>>> [Sat May 4 15:42:07 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time [Sat May 4 15:42:06 CST 2024] <<<<<< start DM-076 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" [Sat May 4 15:42:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:42:07 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker1.toml >>>>>> rpc addr 127.0.0.1:8261 is alive check_task_wrong_arg dmctl test cmd: "check-task" dmctl test cmd: "check-task not_exists_config_file" wait for rpc addr 127.0.0.1:8262 alive the 1-th time pause_relay_wrong_arg dmctl test cmd: "pause-relay wrong_arg" dmctl test cmd: "pause-relay" wait process dm-master.test exit... process dm-master.test already exit resume_relay_wrong_arg dmctl test cmd: "resume-relay wrong_arg" dmctl test cmd: "resume-relay" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/sharding2/source2.yaml" pause_task_wrong_arg dmctl test cmd: "pause-task" dmctl test cmd: "pause-task -s a -s b" got=1 expected=1 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8561 is alive check diff failed 2-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source1.yaml" resume_task_wrong_arg dmctl test cmd: "resume-task" dmctl test cmd: "resume-task -s a -s b" [Sat May 4 15:42:08 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time query_status_wrong_arg dmctl test cmd: "query-status wrong_args_count more_than_one" dmctl test cmd: "query-status -s source-x task-y" start_task_wrong_arg dmctl test cmd: "start-task" wait process dm-worker.test exit... dmctl test cmd: "start-task not_exists_config_file" stop_task_wrong_arg dmctl test cmd: "stop-task" dmctl test cmd: "stop-task -s a -s b" show_ddl_locks_wrong_arg dmctl test cmd: "shard-ddl-lock a b" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "show-ddl-locks a b" purge_relay_wrong_arg dmctl test cmd: "purge-relay wrong_arg" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_2/source2.yaml" dmctl test cmd: "purge-relay" dmctl test cmd: "purge-relay --filename bin-000001 -s mysql-replica-01 -s mysql-replica-02" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:09 CST 2024] <<<<<< test case drop_column_with_index success! >>>>>> start running case: [duplicate_event] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit operate_source_empty_arg dmctl test cmd: "operate-source" dmctl test cmd: "operate-source create not_exists_config_file" [Sat May 4 15:42:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check diff failed 3-th time, retry later dmctl test cmd: "operate-source invalid /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" dmctl test cmd: "operate-source stop /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/source1.yaml" got=2 expected=2 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " transfer_source_empty_arg dmctl test cmd: "transfer-source" dmctl test cmd: "transfer-source source-id" dmctl test cmd: "transfer-source source-id worker-id another-worker" check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:10 CST 2024] <<<<<< finish DM-076 pessimistic >>>>>> [Sat May 4 15:42:10 CST 2024] <<<<<< start DM-076 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" start_relay_empty_arg dmctl test cmd: "start-relay" dmctl test cmd: "start-relay wrong_arg" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:42:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> [Sat May 4 15:42:10 CST 2024] <<<<<< start DM-4215 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" stop_relay_empty_arg dmctl test cmd: "stop-relay" dmctl test cmd: "stop-relay wrong_arg" wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" source-id: mysql-replica-01 server-id: 111111111 enable-gtid: false relay-binlog-name: '' relay-binlog-gtid: '' enable-relay: false from: host: 127.0.0.1 user: root port: 3306 filters: - schema-pattern: "dmctl" table-pattern: "t_1" sql-pattern: ["alter table .* add column aaa int"] action: Ignore - schema-pattern: "dmctl" table-pattern: "t_2" sql-pattern: ["alter table .* add column aaa int"] action: Ignore dmctl test cmd: "operate-source stop /tmp/dm_test/dmctl_basic/source1-wrong.yaml mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source1.yaml" check sync diff for the increment replication check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time finish prepare_data [Sat May 4 15:42:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:42:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-task.yaml " check diff failed 4-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/many_tables/source1.yaml" dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task.yaml " rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/source2.yaml" dmctl test cmd: "operate-source show" [Sat May 4 15:42:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time check sync diff for the increment replication check diff failed 1-th time, retry later [Sat May 4 15:42:13 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master-join5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time check log contain failed 1-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source show" dmctl test cmd: "operate-source show -s mysql-replica-01" start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=1 expected=1 ############################################################ test case for operate tasks bound to a source ############################################################ dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/foo-task.yaml" rpc addr 127.0.0.1:8661 is alive check diff successfully read binlog from relay log failed, and will use remote binlog check diff failed 5-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:14 CST 2024] <<<<<< finish DM-076 optimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 19.07 secs (172322447 bytes/sec) [Pipeline] { [Pipeline] cache wait process dm-worker.test exit... [Sat May 4 15:42:15 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" check log contain failed 2-th time, retry later dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" check diff successfully kill dm-worker1 wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:42:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> check diff failed 6-th time, retry later wait process dm-worker1 exit... wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "pause-task -s xxx" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "resume-task -s xxx" dmctl test cmd: "stop-task -s xxx" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "pause-task -s mysql-replica-01" wait process dm-worker1 exit... process dm-worker1 already exit kill dm-worker2 [Sat May 4 15:42:17 CST 2024] <<<<<< finish DM-077 >>>>>> [Sat May 4 15:42:17 CST 2024] <<<<<< start DM-077 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive check log contain failed 3-th time, retry later check diff failed 7-th time, retry later wait process dm-worker2 exit... dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" 0 in Load unit "progress": "0.00 %", [Sat May 4 15:42:19 CST 2024] <<<<<< finish DM-077 >>>>>> wait process dm-worker2 exit... process dm-worker2 already exit [Sat May 4 15:42:19 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time try to isolate leader dm-master3 kill dm-master3 dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" check log contain failed 4-th time, retry later check diff successfully [Sat May 4 15:42:20 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:42:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master3 exit... [Sat May 4 15:42:20 CST 2024] <<<<<< finish DM-COMPACT >>>>>> dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-02" rpc addr 127.0.0.1:8263 is alive wait for port 8262 offline the 1-th time wait process dm-master3 exit... process dm-master3 already exit [Sat May 4 15:42:21 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master3.toml >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker exit... check diff failed 1-th time, retry later try to isolate follower dm-master2 kill dm-master2 check log contain failed 5-th time, retry later wait process dm-master2 exit... dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" wait for port 8262 offline the 2-th time wait process dm-worker exit... process dm-worker already exit [Sat May 4 15:42:22 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sat May 4 15:42:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master2 exit... check log contain failed 6-th time, retry later [Sat May 4 15:42:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sat May 4 15:42:23 CST 2024] <<<<<< start DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 6.10 secs (538672979 bytes/sec) [Pipeline] { [Pipeline] cache dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" 0 in Load unit "progress": "0.00 %", check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:24 CST 2024] <<<<<< finish DM-078 >>>>>> [Sat May 4 15:42:24 CST 2024] <<<<<< start DM-078 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait process dm-master2 exit... rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:42:24 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/sharding2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/bar-task.yaml" check log contain failed 7-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check diff failed 1-th time, retry later check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive run tidb sql failed 1-th time, retry later wait process dm-master2 exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" check diff successfully check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/baz-task.yaml" wait process dm-master2 exit... dmctl test cmd: "binlog skip test -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "unlock-ddl-lock test-`handle_error`.`tb` --force-remove" run tidb sql failed 2-th time, retry later dmctl test cmd: "query-status -s mysql-replica-02" got=2 expected=2 check sync diff for the increment replication check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 got=2 expected=2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master2 exit... check log contain failed 8-th time, retry later check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:28 CST 2024] <<<<<< finish DM-078 >>>>>> dmctl test cmd: "pause-task -s mysql-replica-01" wait process dm-master.test exit... wait process dm-master2 exit... process dm-master2 already exit [Sat May 4 15:42:28 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-master2.toml >>>>>> "worker1", "worker2", find workers: worker1 for task: test find workers: worker2 for task: test find all workers: worker1 worker2 (total: 2) try to isolate dm-worker1 kill dm-worker1 [Sat May 4 15:42:29 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... 0 in Load unit "progress": "24.83 %", wait process dm-master.test exit... process dm-master.test already exit check log contain failed 9-th time, retry later wait process dm-master.test exit... process dm-master.test already exit run tidb sql failed 3-th time, retry later wait process dm-worker1 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "pause-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" [Sat May 4 15:42:31 CST 2024] <<<<<< finish DM-079 >>>>>> [Sat May 4 15:42:31 CST 2024] <<<<<< start DM-079 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "resume-task -s mysql-replica-01" wait process dm-worker1 exit... process dm-worker1 already exit [Sat May 4 15:42:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker1.toml >>>>>> try to isolate dm-worker2 kill dm-worker2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/GetSessionVariableFailed=return("sql_mode,1152") [Sat May 4 15:42:31 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... check log contain failed 10-th time, retry later wait process dm-worker2 exit... run tidb sql failed 4-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:42:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "resume-task -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:32 CST 2024] <<<<<< test case sharding2 success! >>>>>> start running case: [ha] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit import prepare data start DM worker and master [Sat May 4 15:42:33 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master1.toml >>>>>> [Sat May 4 15:42:33 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master2.toml >>>>>> [Sat May 4 15:42:32 CST 2024] <<<<<< finish DM-079 >>>>>> dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-01" wait process dm-worker2 exit... process dm-worker2 already exit [Sat May 4 15:42:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases3_1/conf/dm-worker2.toml >>>>>> dmctl test cmd: "pause-task test" got=3 expected=3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_dm_integration_test-1875/tiflow-dm) 3285469696 bytes in 6.41 secs (512180621 bytes/sec) [Pipeline] { wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { dmctl test cmd: "resume-task test" got=3 expected=3 [Pipeline] sh [Pipeline] sh [Sat May 4 15:42:33 CST 2024] <<<<<< start DM-080 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 11-th time, retry later + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 4 15:41 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:41 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:41 .github -rw-r--r--. 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:42 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 15:42 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:42 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 4 15:42 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:42 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:42 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 examples -rw-r--r--. 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 4 15:42 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:42 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:42 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:42 tests -rw-r--r--. 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break [Pipeline] sh run tidb sql failed 5-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/relay_interrupt/1/source1.yaml" query status, relay log failed dmctl test cmd: "query-status -s mysql-replica-01" got=1 expected=1 got=1 expected=1 start task and query status, task and relay have error message + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 4 15:42 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:42 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 .github -rw-r--r--. 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:42 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 15:42 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:42 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 4 15:42 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:42 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:42 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 examples -rw-r--r--. 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 4 15:42 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:42 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:42 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:42 tests -rw-r--r--. 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "stop-task -s mysql-replica-02" + pwd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow + ls -alh total 392K drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 .. -rw-r--r--. 1 jenkins jenkins 2.4K May 4 15:27 .codecov.yml lrwxrwxrwx. 1 jenkins jenkins 10 May 4 15:42 .dockerignore -> .gitignore -rw-r--r--. 1 jenkins jenkins 162 May 4 15:27 .editorconfig drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:42 .git drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 .github -rw-r--r--. 1 jenkins jenkins 985 May 4 15:27 .gitignore -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 .golangci.yml -rw-r--r--. 1 jenkins jenkins 156 May 4 15:27 CODE_OF_CONDUCT.md -rw-r--r--. 1 jenkins jenkins 3.5K May 4 15:27 CONTRIBUTING.md -rw-r--r--. 1 jenkins jenkins 12K May 4 15:27 LICENSE -rw-r--r--. 1 jenkins jenkins 25K May 4 15:27 Makefile -rw-r--r--. 1 jenkins jenkins 2.7K May 4 15:27 Makefile.engine -rw-r--r--. 1 jenkins jenkins 775 May 4 15:27 OWNERS -rw-r--r--. 1 jenkins jenkins 124 May 4 15:27 OWNERS_ALIASES -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 README.md -rw-r--r--. 1 jenkins jenkins 2.1K May 4 15:27 README_DM.md -rw-r--r--. 1 jenkins jenkins 5.2K May 4 15:27 README_Engine.md -rw-r--r--. 1 jenkins jenkins 2.8K May 4 15:27 README_TiCDC.md -rw-r--r--. 1 jenkins jenkins 1.5K May 4 15:27 SECURITY.md drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:42 bin drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 15:42 cdc drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 15:42 cdcv2 drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:42 cmd drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 deployments drwxr-sr-x. 26 jenkins jenkins 4.0K May 4 15:42 dm drwxr-sr-x. 7 jenkins jenkins 4.0K May 4 15:42 docs drwxr-sr-x. 13 jenkins jenkins 4.0K May 4 15:42 engine -rwxr-xr-x. 1 jenkins jenkins 27K May 4 15:27 errors.toml drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 examples -rw-r--r--. 1 jenkins jenkins 21K May 4 15:27 go.mod -rw-r--r--. 1 jenkins jenkins 158K May 4 15:27 go.sum drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 metrics drwxr-sr-x. 44 jenkins jenkins 4.0K May 4 15:42 pkg drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:42 proto drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 15:42 scripts drwxr-sr-x. 5 jenkins jenkins 4.0K May 4 15:42 tests -rw-r--r--. 1 jenkins jenkins 582 May 4 15:27 third-party-license.txt drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 15:42 tools + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break + set +e + for i in '{1..90}' + mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent mysqld is alive + '[' 0 -eq 0 ']' + set -e + break wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-02" dmctl test cmd: "transfer-source mysql-replica-01 worker1" 0 in Load unit "progress": "51.09 %", dmctl test cmd: "transfer-source mysql-replica-01 worker2" kill worker2 dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later waiting for asynchronous relay and subtask to be started check log contain failed 12-th time, retry later run tidb sql failed 6-th time, retry later rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive [Sat May 4 15:42:35 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... rpc addr 127.0.0.1:8262 is alive operate mysql config to worker dmctl test cmd: "operate-source create /tmp/dm_test/ha/source1.yaml" dmctl test cmd: "binlog skip test" wait process worker2 exit... process worker2 already exit dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" restart worker2 [Sat May 4 15:42:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status -s mysql-replica-01" dmctl test cmd: "query-status" dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later check log contain failed 13-th time, retry later run tidb sql failed 7-th time, retry later reset go failpoints, and need restart dm-worker then resume task, task will recover success dmctl test cmd: "stop-task test" [Sat May 4 15:42:37 CST 2024] <<<<<< finish DM-080 pessimistic >>>>>> [Sat May 4 15:42:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1" [Sat May 4 15:42:38 CST 2024] <<<<<< start DM-081 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha/source2.yaml" dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "start-relay -s mysql-replica-01 worker2" dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "stop-relay -s mysql-replica-01" wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:42:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "stop-task test" [Sat May 4 15:42:39 CST 2024] <<<<<< finish DM-4215 pessimistic >>>>>> [Sat May 4 15:42:39 CST 2024] <<<<<< start DM-4215 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 14-th time, retry later 0 in Load unit "progress": "76.75 %", pause_relay_success dmctl test cmd: "pause-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 8-th time, retry later got=3 expected=3 got=2 expected=2 dmctl test cmd: "resume-relay -s mysql-replica-01 -s mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" check diff failed 1-th time, retry later dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl_check_task dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task2.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 1" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 100 -w 1" got=2 expected=2 [Sat May 4 15:42:41 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8461 alive the 1-th time dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml -e 0" got=2 expected=2 got=2 expected=2 dmctl test cmd: "binlog skip test" check log contain failed 15-th time, retry later run tidb sql failed 9-th time, retry later dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task5.yaml" dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task6.yaml" check_task_optimistic dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task4.yaml" [Sat May 4 15:42:42 CST 2024] <<<<<< finish DM-4215 optimistic >>>>>> check_task_only_warning dmctl test cmd: "check-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" rpc addr 127.0.0.1:8461 is alive ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 1-th time, got value: '' dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:42 CST 2024] <<<<<< finish DM-081 pessimistic >>>>>> [Sat May 4 15:42:43 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" check log contain failed 16-th time, retry later [Sat May 4 15:42:43 CST 2024] <<<<<< start DM-082 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" run tidb sql failed 10-th time, retry later ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 2-th time, got value: '' dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/only_warning.yaml" check task with empty unit config dmctl test cmd: "check-task /tmp/empty-dump.yaml" --- name: empty-unit-task task-mode: all target-database: host: "127.0.0.1" port: 4000 user: "root" password: "" mysql-instances: - source-id: "mysql-replica-01" mydumper-config-name: "global" loader-config-name: "global" syncer-config-name: "global" block-allow-list: "instance" mydumpers: global: threads: 4 loaders: global: syncers: global: worker-count: 32 block-allow-list: instance: do-dbs: ["dmctl"] do-tables: - db-name: "dmctl" tbl-name: "~^t_[\\d]+"dmctl test cmd: "check-task /tmp/empty-load.yaml" [Pipeline] } [Pipeline] } [Pipeline] } dmctl test cmd: "check-task /tmp/empty-sync.yaml" start task with empty unit config dmctl test cmd: "start-task /tmp/empty-cfg.yaml" dmctl test cmd: "query-status test" [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout got=2 expected=2 dmctl test cmd: "query-status test" [Pipeline] sh ./tests/_utils/check_metric: line 20: [: 1: unary operator expected wait for valid metric of dm_master_worker_state{worker="worker1"} for 3-th time, got value: '' got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:5043 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 [Pipeline] sh 0 in Load unit "progress": "100.00 %", dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:5548 alter table handle_error.tb1 add column c varchar(20);" check diff failed 1-th time, retry later + '[' TLS_GROUP == TLS_GROUP ']' + echo 'run tls test' run tls test + echo 'copy mysql certs' copy mysql certs + sudo mkdir -p /var/lib/mysql + sudo chmod 777 /var/lib/mysql + sudo chown -R 1000:1000 /var/lib/mysql + sudo cp -r /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ca-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/ca.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/client-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/client-key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/private_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/public_key.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/server-cert.pem /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/mysql-ssl/server-key.pem /var/lib/mysql/ + sudo chown -R 1000:1000 /var/lib/mysql/ca-key.pem /var/lib/mysql/ca.pem /var/lib/mysql/client-cert.pem /var/lib/mysql/client-key.pem /var/lib/mysql/private_key.pem /var/lib/mysql/public_key.pem /var/lib/mysql/server-cert.pem /var/lib/mysql/server-key.pem + ls -alh /var/lib/mysql/ total 48K drwxrwxrwx. 2 jenkins jenkins 4.0K May 4 15:42 . drwxr-xr-x. 1 root root 4.0K May 4 15:42 .. -rw-------. 1 jenkins jenkins 1.7K May 4 15:42 ca-key.pem -rw-r--r--. 1 jenkins jenkins 1.1K May 4 15:42 ca.pem -rw-r--r--. 1 jenkins jenkins 1.1K May 4 15:42 client-cert.pem -rw-------. 1 jenkins jenkins 1.7K May 4 15:42 client-key.pem -rw-------. 1 jenkins jenkins 1.7K May 4 15:42 private_key.pem -rw-r--r--. 1 jenkins jenkins 451 May 4 15:42 public_key.pem -rw-r--r--. 1 jenkins jenkins 1.1K May 4 15:42 server-cert.pem -rw-------. 1 jenkins jenkins 1.7K May 4 15:42 server-key.pem + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ [Pipeline] sh dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" check log contain failed 17-th time, retry later + make dm_integration_test_in_group GROUP=TLS_GROUP + '[' TLS_GROUP == G10 ']' + echo 'run G10 test' run G10 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ run tidb sql failed 11-th time, retry later dmctl test cmd: "config task empty-unit-task" dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/empty-unit-task.yaml" [Sat May 4 15:42:46 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Sat May 4 15:42:46 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" metric dm_master_worker_state{worker="worker1"} has invalid value start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml " /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + make dm_integration_test_in_group GROUP=G10 + '[' TLS_GROUP == G11 ']' + echo 'run G11 test' run G11 test + export PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PATH=/usr/local/go/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + mkdir -p ./dm/tests/bin + cp -r ./bin/dm-test-tools/check_exit_safe_binlog ./bin/dm-test-tools/check_master_http_apis ./bin/dm-test-tools/check_master_online ./bin/dm-test-tools/check_master_online_http ./bin/dm-test-tools/check_worker_online ./dm/tests/bin/ dmctl test cmd: "check-task /tmp/dm_test/dmctl_basic/dm-task-error-database-config.yaml" dmctl_start_task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task3.yaml --start-time '20060102 150405'" /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt + make dm_integration_test_in_group GROUP=G11 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task7.yaml --start-time '2006-01-02 15:04:05'" /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/sync_diff_inspector /usr/bin/mysql /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/minio install python requirments for test pip install --user -q -r ./dm/tests/requirements.txt /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh TLS_GROUP Run cases: tls ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G10 Run cases: start_task print_status http_apis new_relay all_mode ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) dmctl test cmd: "query-status test" got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase use sync_diff_inspector to check increment data /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-master.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-worker.test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/bin/dm-syncer.test cd dm && ln -sf ../bin . cd dm && ./tests/run_group.sh G11 Run cases: validator_basic dm_syncer shardddl_optimistic slow_relay_writer sql_mode sync_collation ... rm: cannot remove '/tmp/dm_test/tidb.toml': No such file or directory Starting TiDB on port 4000 Verifying TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check log contain failed 18-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 check diff failed 1-th time, retry later run tidb sql failed 12-th time, retry later dmctl test cmd: "query-status test_incremental_no_source_meta" got=3 expected=3 dmctl test cmd: "stop-task test_incremental_no_source_meta" dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:6850 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml " check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:47 CST 2024] <<<<<< finish DM-082 pessimistic >>>>>> got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:7521 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" [Sat May 4 15:42:48 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> [Sat May 4 15:42:48 CST 2024] <<<<<< start DM-4216 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" [Sat May 4 15:42:48 CST 2024] <<<<<< start DM-085 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" start task after restarted dm-worker dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-task.yaml" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully read binlog from relay log failed, and will use remote binlog check diff successfully dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "stop-relay -s mysql-replica-01 worker2" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [tls] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/tls/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" version() 5.7.26-log version() 8.0.21 start running case: [validator_basic] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh... Verbose mode = false --> full mode, check we validate different data types(gtid=false, relay=false) 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start dm-worker3 and kill dm-worker2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. version() 5.7.26-log version() 8.0.21 start running case: [start_task] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:42:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "stop-relay -s mysql-replica-02 worker2" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ check log contain failed 19-th time, retry later wait process dm-worker.test exit... run tidb sql failed 13-th time, retry later got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:8915 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" got=1 expected=1 add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:9494 alter table handle_error.tb1 add column c varchar(20);" check diff failed 1-th time, retry later 0 in Sync unit rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:42:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:42:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:42:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/relay_interrupt/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker2 exit... wait process dmctl_basic/worker1 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/source1.yaml" check log contain failed 20-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sat May 4 15:42:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker2 exit... process dm-worker2 already exit [Sat May 4 15:42:51 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time run tidb sql failed 14-th time, retry later wait process dmctl_basic/worker1 exit... process dmctl_basic/worker1 already exit rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" check diff successfully rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-02 worker2" [Sat May 4 15:42:52 CST 2024] <<<<<< finish DM-085 pessimistic >>>>>> [Sat May 4 15:42:52 CST 2024] <<<<<< start DM-085 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" wait process dmctl_basic/worker2 exit... got=2 expected=2 dmctl test cmd: "stop-task test" [Sat May 4 15:42:53 CST 2024] <<<<<< finish DM-4216 pessimistic >>>>>> [Sat May 4 15:42:53 CST 2024] <<<<<< start DM-4216 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" check log contain failed 21-th time, retry later wait process dmctl_basic/worker2 exit... process dmctl_basic/worker2 already exit dmctl test cmd: "query-status" run tidb sql failed 15-th time, retry later got=2 expected=2 got=2 expected=2 [Sat May 4 15:42:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "config source mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:10980 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" [Sat May 4 15:42:54 CST 2024] <<<<<< finish DM-085 optimistic >>>>>> got=1 expected=1 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:11467 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:42:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=3 expected=3 dmctl test cmd: "stop-task test" [Sat May 4 15:42:55 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Sat May 4 15:42:55 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Sat May 4 15:42:55 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sat May 4 15:42:55 CST 2024] <<<<<< finish DM-4216 optimistic >>>>>> check log contain failed 22-th time, retry later dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-04 09:42:53'" [Sat May 4 15:42:55 CST 2024] <<<<<< start DM-086 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-pessimistic-2.yaml --remove-meta" check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:56 CST 2024] <<<<<< finish DM-COMPACT_USE_DOWNSTREAM_SCHEMA >>>>>> wait for dm to sync use sync_diff_inspector to check data now! check diff successfully [Sat May 4 15:42:46 CST 2024] <<<<<< finish test_kill_master_in_sync >>>>>> [Sat May 4 15:42:46 CST 2024] <<<<<< start test_kill_worker_in_sync >>>>>> [Sat May 4 15:42:46 CST 2024] <<<<<< start test_running >>>>>> 2 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive config dmctl test cmd: "config" dmctl test cmd: "config hihi haha" dmctl test cmd: "config task haha" dmctl test cmd: "config master haha" [Sat May 4 15:42:56 CST 2024] <<<<<< start DM-4219 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "config worker haha" dmctl test cmd: "config source haha" got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=false, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "get-config haha" dmctl test cmd: "get-config haha hihi" wait process dm-master.test exit... dmctl test cmd: "stop-task test" dmctl test cmd: "stop-relay -s mysql-replica-01" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" wait process dm-worker exit... dmctl test cmd: "get-config master haha" dmctl test cmd: "get-config worker haha" dmctl test cmd: "get-config source haha" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status -s mysql-replica-01" [Sat May 4 15:42:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "get-config task haha" dmctl test cmd: "get-task-config haha" check log contain failed 23-th time, retry later [Sat May 4 15:42:57 CST 2024] <<<<<< finish DM-086 pessimistic >>>>>> [Sat May 4 15:42:57 CST 2024] <<<<<< start DM-086 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/single-source-optimistic-2.yaml --remove-meta" start incremental_data dmctl test cmd: "get-task-config haha hihi" dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" wait process dm-master.test exit... dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" dmctl test cmd: "query-status test" wait process dm-worker exit... process dm-worker already exit [Sat May 4 15:42:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sat May 4 15:42:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Sat May 4 15:42:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" run tidb sql failed 1-th time, retry later wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker2" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-05-04 09:42:53'" dmctl test cmd: "operate-source stop mysql-replica-01" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source1.yaml" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:42:59 CST 2024] <<<<<< finish DM-086 optimistic >>>>>> rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sat May 4 15:42:59 CST 2024] <<<<<< start DM-MULTIPLE_ROWS >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" wait process dm-master.test exit... wait process dm-worker.test exit... check log contain failed 24-th time, retry later dmctl test cmd: "operate-source stop mysql-replica-02" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_basic/get_source2.yaml" dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '1995-03-07 01:02:03'" dmctl test cmd: "query-status test" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=1 expected=1 [Sat May 4 15:43:00 CST 2024] <<<<<< start DM-094 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker.test exit... check master alive dmctl test cmd: "list-member" start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" check log contain failed 1-th time, retry later dmctl test cmd: "stop-task test" [Sat May 4 15:43:00 CST 2024] <<<<<< finish DM-4219 pessimistic >>>>>> [Sat May 4 15:43:00 CST 2024] <<<<<< start DM-4219 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait for the task to be scheduled and keep paused HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive resume task before kill and restart dm-worker dmctl test cmd: "resume-task test" wait process dm-master.test exit... start dm-worker2 and kill dm-worker3 dmctl test cmd: "start-task /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2037-12-12 01:02:03'" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 25-th time, retry later got=2 expected=2 check diff successfully wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -s mysql-replica-01 alter table handle_error.tb1 add column c varchar(20);" wait process dm-worker3 exit... dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test -s mysql-replica-02 alter table handle_error.tb1 add column c varchar(20);" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive check log contain failed 2-th time, retry later wait process dm-master.test exit... test http and api interface Release Version: v8.2.0-alpha-72-g7dd6241bd Git Commit Hash: 7dd6241bd56139ff08ada1b98fc467877b87f31a Git Branch: HEAD UTC Build Time: 2024-05-04 07:28:25 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:03 CST 2024] <<<<<< finish DM-4219 optimistic >>>>>> wait process dm-master exit... wait process dm-worker3 exit... process dm-worker3 already exit [Sat May 4 15:43:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time { "result": true, "msg": "", "members": [ { "leader": { "msg": "", "name": "master2", "addr": "127.0.0.1:8361" } }, { "master": { "msg": "", "masters": [ { "name": "master1", "memberID": "10068700568285020701", "alive": true, "peerURLs": [ "https://127.0.0.1:8291" ], "clientURLs": [ "https://127.0.0.1:8261" ] }, { "name": "master2", "memberID": "518295880577331676", "alive": true, "peerURLs": [ "https://127.0.0.1:8292" ], "clientURLs": [ "https://127.0.0.1:8361" ] }, { "name": "master3", "memberID": "6800138986734450252", "alive": true, "peerURLs": [ "https://127.0.0.1:8293" ], "clientURLs": [ "https://127.0.0.1:8461" ] } ] } }, { "worker": { "msg": "", "workers": [ { "name": "worker1", "addr": "127.0.0.1:8262", "stage": "bound", "source": "mysql-replica-01" } ] } } ] } rpc addr 127.0.0.1:8261 is alive use common name not in 'cert-allowed-cn' should not request success wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" check log contain failed 26-th time, retry later wait process dm-master.test exit... [Sat May 4 15:43:04 CST 2024] <<<<<< finish DM-094 pessimistic >>>>>> [Sat May 4 15:43:04 CST 2024] <<<<<< start DM-094 optimistic >>>>>> wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:04 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sat May 4 15:43:04 CST 2024] <<<<<< start DM-4220 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-worker.test exit... wait process dm-master exit... process dm-master already exit [Sat May 4 15:43:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit [Sat May 4 15:43:05 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 3-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:05 CST 2024] <<<<<< test case relay_interrupt success! >>>>>> start running case: [safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time run tidb sql failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check log contain failed 27-th time, retry later got=2 expected=2 dmctl test cmd: "binlog revert test" dmctl test cmd: "query-status test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:16594" got=2 expected=2 check diff successfully wait process dm-worker.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:17605" dmctl test cmd: "query-status test" dmctl test cmd: "stop-task test" got=3 expected=3 dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml" got=2 expected=2 dmctl test cmd: "binlog revert test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:16787" wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait for rpc addr 127.0.0.1:8261 alive the 4-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:06 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:17824" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:16787" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:17824" wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" run tidb sql failed 2-th time, retry later rpc addr 127.0.0.1:8261 is alive check log contain failed 28-th time, retry later dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait for rpc addr 127.0.0.1:8261 alive the 5-th time got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:43:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again run tidb sql failed 1-th time, retry later [Sat May 4 15:43:08 CST 2024] <<<<<< finish DM-094 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 6-th time wait process worker1 exit... got=2 expected=2 wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" [Sat May 4 15:43:09 CST 2024] <<<<<< start DM-095 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 7-th time dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:16787" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:17824" got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=false) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process worker1 exit... run tidb sql failed 3-th time, retry later dmctl test cmd: "resume-task test" got=2 expected=2 finish incremental_data check diff 1 wait process dm-master.test exit... [Sat May 4 15:43:09 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time check log contain failed 29-th time, retry later dmctl test cmd: "binlog revert test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:17824" dmctl test cmd: "binlog revert test" dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 8-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time [Sat May 4 15:43:10 CST 2024] <<<<<< finish DM-4220 pessimistic >>>>>> [Sat May 4 15:43:10 CST 2024] <<<<<< start DM-4220 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "pause-task test" wait process dm-master.test exit... wait process worker1 exit... dmctl test cmd: "resume-task test" got=3 expected=3 check diff failed 1-th time, retry later wait process dm-master.test exit... wait process worker1 exit... run tidb sql failed 4-th time, retry later check diff successfully wait for rpc addr 127.0.0.1:8261 alive the 9-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:11 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check log contain failed 30-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit got=2 expected=2 dmctl test cmd: "binlog revert test" wait process dm-master.test exit... check diff failed 1-th time, retry later dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:19218" wait process worker1 exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:20496" dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 10-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/source1.yaml" wait and check task running HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive [Sat May 4 15:43:12 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog revert test" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:19411" wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:20715" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:19411" dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:20715" wait process worker1 exit... wait process dm-master.test exit... run tidb sql failed 5-th time, retry later wait for rpc addr 127.0.0.1:8261 alive the 11-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "transfer-source mysql-replica-01 worker3" dmctl test cmd: "list-member --name worker3" wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-master.test exit... got=1 expected=1 query-status from all dm-master dmctl test cmd: "query-status test" run tidb sql failed 1-th time, retry later got=3 expected=3 dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "stop-task test" check diff failed 1-th time, retry later got=3 expected=3 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 12-th time got=3 expected=3 join new dm-master and query-status [Sat May 4 15:43:14 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master4.toml >>>>>> wait for rpc addr 127.0.0.1:8561 alive the 1-th time wait process worker1 exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:15 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... [Sat May 4 15:43:15 CST 2024] <<<<<< finish DM-095 pessimistic >>>>>> [Sat May 4 15:43:15 CST 2024] <<<<<< start DM-095 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" run tidb sql failed 6-th time, retry later check diff successfully [Sat May 4 15:43:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8561 is alive dmctl test cmd: "query-status test" wait process worker1 exit... wait for rpc addr 127.0.0.1:8261 alive the 13-th time got=3 expected=3 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:19411" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:16 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-master.test exit... dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:20715" dmctl test cmd: "binlog revert test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:20715" dmctl test cmd: "binlog revert test" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" check diff successfully [2024/05/04 15:42:12.744 +08:00] [INFO] [dumpling.go:111] ["create dumpling"] [task=test] [unit=dump] [config="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"auto\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"\",\"ServerInfo\":{\"ServerType\":0,\"ServerVersion\":null,\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/04 15:42:12.872 +08:00] [INFO] [dump.go:151] ["begin to run Dump"] [task=test] [unit=dump] [conf="{\"s3\":{\"endpoint\":\"\",\"region\":\"\",\"storage-class\":\"\",\"sse\":\"\",\"sse-kms-key-id\":\"\",\"acl\":\"\",\"access-key\":\"\",\"secret-access-key\":\"\",\"session-token\":\"\",\"provider\":\"\",\"force-path-style\":false,\"use-accelerate-endpoint\":false,\"role-arn\":\"\",\"external-id\":\"\",\"object-lock-enabled\":false},\"gcs\":{\"endpoint\":\"\",\"storage-class\":\"\",\"predefined-acl\":\"\",\"credentials-file\":\"\"},\"azblob\":{\"endpoint\":\"\",\"account-name\":\"\",\"account-key\":\"\",\"access-tier\":\"\",\"sas-token\":\"\",\"encryption-scope\":\"\",\"encryption-key\":\"\"},\"SpecifiedTables\":false,\"AllowCleartextPasswords\":false,\"SortByPk\":true,\"NoViews\":true,\"NoSequences\":true,\"NoHeader\":false,\"NoSchemas\":false,\"NoData\":false,\"CompleteInsert\":true,\"TransactionalConsistency\":true,\"EscapeBackslash\":true,\"DumpEmptyDatabase\":true,\"PosAfterConnect\":false,\"CompressType\":0,\"Host\":\"127.0.0.1\",\"Port\":3306,\"Threads\":4,\"User\":\"root\",\"Security\":{\"CAPath\":\"\",\"CertPath\":\"\",\"KeyPath\":\"\"},\"LogLevel\":\"\",\"LogFile\":\"\",\"LogFormat\":\"\",\"OutputDirPath\":\"./dumped_data.test\",\"StatusAddr\":\"\",\"Snapshot\":\"\",\"Consistency\":\"flush\",\"CsvNullValue\":\"\\\\N\",\"SQL\":\"\",\"CsvSeparator\":\",\",\"CsvDelimiter\":\"\\\"\",\"CsvLineTerminator\":\"\\r\\n\",\"Databases\":null,\"Where\":\"\",\"FileType\":\"sql\",\"ServerInfo\":{\"ServerType\":1,\"ServerVersion\":\"5.7.26-log\",\"HasTiKV\":false},\"Rows\":200000,\"ReadTimeout\":0,\"TiDBMemQuotaQuery\":0,\"FileSize\":0,\"StatementSize\":100,\"SessionParams\":{\"time_zone\":\"+06:00\"},\"Tables\":null,\"CollationCompatible\":\"loose\",\"CsvOutputDialect\":0,\"IOTotalBytes\":null,\"Net\":\"\"}"] [2024/05/04 15:42:22.763 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14321] [2024/05/04 15:42:32.764 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14321] [2024/05/04 15:42:42.765 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=14321] [2024/05/04 15:42:52.766 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=9415419] [2024/05/04 15:43:02.766 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=10245174] [2024/05/04 15:43:12.767 +08:00] [DEBUG] [syncer.go:362] [IOTotalBytes] [task=test] [unit="binlog replication"] [IOTotalBytes=20038162] check diff 2 dmctl test cmd: "stop-task test" wait process worker1 exit... got=3 expected=3 check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:43:17 CST 2024] <<<<<< finish DM-4220 optimistic >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 14-th time check diff successfully rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source2.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sat May 4 15:43:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait process dm-master.test exit... [Sat May 4 15:43:17 CST 2024] <<<<<< finish DM-095 optimistic >>>>>> wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 1 [Sat May 4 15:43:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8263 alive the 1-th time check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " wait process worker1 exit... [Sat May 4 15:43:18 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" wait for rpc addr 127.0.0.1:8261 alive the 15-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task-relay.yaml --remove-meta" check diff successfully check diff successfully 40 11 11 51 dmctl test cmd: "stop-task test" [Sat May 4 15:43:18 CST 2024] <<<<<< finish DM-MULTIPLE_ROWS >>>>>> wait process dm-master.test exit... [Sat May 4 15:43:18 CST 2024] <<<<<< start DM-096 pessimistic >>>>>> rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" [Sat May 4 15:43:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process worker1 exit... process worker1 already exit wait for rpc addr 127.0.0.1:8261 alive the 16-th time wait process dm-worker.test exit... wait process dm-worker exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "start-relay -s mysql-replica-02 worker2" dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:43:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:21842" wait for rpc addr 127.0.0.1:8261 alive the 17-th time dmctl test cmd: "config source mysql-replica-01" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:43:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process worker2 exit... dmctl test cmd: "pause-task test" dmctl test cmd: "resume-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:23387" wait process dm-master.test exit... got=3 expected=3 check diff failed 1-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-worker exit... process dm-worker already exit [Sat May 4 15:43:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>> [Sat May 4 15:43:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" got=2 expected=2 dmctl test cmd: "query-status -s mysql-replica-02" got=1 expected=1 check diff failed 1-th time, retry later [Sat May 4 15:43:20 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Sat May 4 15:43:20 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process worker2 exit... process worker2 already exit dmctl test cmd: "list-member --worker" got=2 expected=2 [Sat May 4 15:43:21 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 18-th time rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive [Sat May 4 15:43:21 CST 2024] <<<<<< start DM-CAUSALITY >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" [Sat May 4 15:43:21 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8661 alive the 1-th time wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 19-th time dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8661 is alive dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:43:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 --> full mode, check we validate different data types(gtid=true, relay=true) 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "query-status test" got=2 expected=2 got=3 expected=3 check diff failed 2-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:23811" check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:43:22 CST 2024] <<<<<< finish DM-096 pessimistic >>>>>> [Sat May 4 15:43:22 CST 2024] <<<<<< start DM-096 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:25545" wait for rpc addr 127.0.0.1:8261 alive the 20-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "list-member --worker" got=2 expected=2 dmctl test cmd: "config task test --path /tmp/dm_test/dmctl_basic/get_task.yaml" dmctl test cmd: "config master master1 --path /tmp/dm_test/dmctl_basic/master/dm-master.toml" check diff successfully [Sat May 4 15:43:23 CST 2024] <<<<<< finish DM-CAUSALITY >>>>>> [Sat May 4 15:43:23 CST 2024] <<<<<< start DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog skip test" wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "config worker worker1 --path /tmp/dm_test/dmctl_basic/worker1/dm-worker.toml" got=1 expected=1 dmctl test cmd: "config worker worker2 --path /tmp/dm_test/dmctl_basic/worker2/dm-worker.toml" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "config source mysql-replica-01 --path /tmp/dm_test/dmctl_basic/get_source1.yaml" dmctl test cmd: "config source mysql-replica-02 --path /tmp/dm_test/dmctl_basic/get_source2.yaml" [Sat May 4 15:43:23 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> wait process dm-worker.test exit... [Sat May 4 15:43:23 CST 2024] <<<<<< start DM-4185 pessimistic >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 check alive timeout check data check diff successfully ============================== test_master_ha_when_enable_tidb_and_only_ca_source_tls success ================================== 3 dm-master alive 1 dm-worker alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... show_ddl_locks_no_locks dmctl test cmd: "shard-ddl-lock test" dmctl test cmd: "show-ddl-locks test" 0 dm-syncer alive wait process dm-master.test exit... process dm-master.test already exit check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" dmctl test cmd: "query-status" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=1 expected=1 run tidb sql failed 1-th time, retry later dmctl test cmd: "resume-task test" got=3 expected=3 wait process dm-worker.test exit... dmctl test cmd: "pause-task test" check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" got=3 expected=3 got=2 expected=2 dmctl master_addr dmctl test cmd: "list-member" check diff failed 3-th time, retry later wait process dm-master.test exit... [Sat May 4 15:43:25 CST 2024] <<<<<< finish DM-096 optimistic >>>>>> wait process dm-master.test exit... wait process dm-worker.test exit... dmctl test cmd: "list-member" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:43:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchTargetDoTablesFailed=return(1152) dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:26038" dmctl test cmd: "query-status test" wait process dm-master.test exit... [Sat May 4 15:43:26 CST 2024] <<<<<< start DM-097 pessimistic >>>>>> wait process dm-master.test exit... wait process dm-worker.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:27703" dmctl test cmd: "query-status test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:43:26 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" [Sat May 4 15:43:26 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=3 expected=3 dmctl test cmd: "stop-task test" [Sat May 4 15:43:26 CST 2024] <<<<<< finish DM-4185 pessimistic >>>>>> [Sat May 4 15:43:26 CST 2024] <<<<<< start DM-4185 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" [Sat May 4 15:43:27 CST 2024] <<<<<< finish DM-CAUSALITY_USE_DOWNSTREAM_SCHEMA >>>>>> [Sat May 4 15:43:27 CST 2024] <<<<<< start DM-UpdateBARule optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully dmctl test cmd: "query-status test" got=1 expected=1 rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=2 expected=2 [Sat May 4 15:43:27 CST 2024] <<<<<< START DM-MASTER on port 8761, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-master6.toml >>>>>> wait for rpc addr 127.0.0.1:8761 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait process dm-master.test exit... check diff failed 4-th time, retry later wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=2 expected=2 wait process tidb-server exit... process tidb-server already exit wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8761 is alive dmctl test cmd: "query-status test" dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:28265" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-master exit... got=2 expected=2 dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:29861" dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=0 expected=1 command: query-status test fail to try sync the optimistic shard ddl lock count: 0 != expected: 1, failed the 0-th time, will retry again got=3 expected=3 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/0/source1.yaml" check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" success success [Sat May 4 15:43:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sat May 4 15:43:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master exit... process dm-master already exit [Sat May 4 15:43:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again wait process dm-master.test exit... wait process dm-master.test exit... check diff failed 5-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sat May 4 15:43:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" got=3 expected=3 wait process dm-worker.test exit... dmctl test cmd: "stop-task test" [Sat May 4 15:43:31 CST 2024] <<<<<< finish DM-4185 optimistic >>>>>> got=0 expected=1 command: query-status test fail to try sync the optimistic shard ddl lock count: 0 != expected: 1, failed the 1-th time, will retry again dmctl test cmd: "list-member" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time check diff successfully binlog_pos: 2356 relay_log_size: 2325 dmctl test cmd: "config source mysql-replica-01" rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" wait process dm-master.test exit... [Sat May 4 15:43:32 CST 2024] <<<<<< start DM-4201 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_2/conf/single-source-no-sharding.yaml --remove-meta" got=1 expected=1 got=1 expected=1 dmctl test cmd: "validation status test" got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... check diff successfully dmctl test cmd: "query-status test" binlog_pos: 2356 relay_log_size: 2356 ============== run_with_prepared_source_config success =================== wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "stop-task test" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sat May 4 15:43:33 CST 2024] <<<<<< finish DM-097 pessimistic >>>>>> [Sat May 4 15:43:33 CST 2024] <<<<<< start DM-097 optimistic >>>>>> got=1 expected=1 dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /tmp/dm_test/shardddl1/task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit [Sat May 4 15:43:34 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time kill dm-master1 wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:30234" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-master1 exit... wait process dm-master.test exit... dmctl test cmd: "show-ddl-locks" got=1 expected=1 restart dm-master wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls [Sat May 4 15:43:35 CST 2024] <<<<<< finish DM-4201 >>>>>> dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-master.test exit... process dm-master.test already exit wait for rpc addr 127.0.0.1:8261 alive the 2-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 --> fast mode, check we validate different data types 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master1 exit... process dm-master1 already exit kill dm-master2 wait process dm-master.test exit... 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master exit... wait process dm-master exit... wait process dm-worker.test exit... wait process dm-master2 exit... wait process dm-master.test exit... dmctl test cmd: "list-member" rpc addr 127.0.0.1:8262 is alive wait process dm-master.test exit... dmctl test cmd: "start-relay -s mysql-replica-01" dmctl test cmd: "start-relay -s mysql-replica-01 worker1" wait process dm-master exit... process dm-master already exit wait process dm-master exit... process dm-master already exit [Sat May 4 15:43:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success dmctl test cmd: "stop-relay -s mysql-replica-01 worker1" dmctl test cmd: "stop-relay -s mysql-replica-01" wait process dm-master2 exit... process dm-master2 already exit initial cluster of dm-masters have been killed now we will check whether joined masters can work normally dmctl test cmd: "stop-task test" success finish running run safe mode recover case 1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-relay -s mysql-replica-01 worker1" dmctl test cmd: "query-status test" wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... dmctl test cmd: "start-relay -s mysql-replica-02 worker2" wait process dm-master.test exit... [Sat May 4 15:43:39 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... dmctl test cmd: "resume-task test" dmctl test cmd: "query-status -s mysql-replica-01,mysql-replica-02" wait process dm-worker.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=3 expected=3 got=4 expected=4 check diff successfully dmctl test cmd: "pause-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" [Sat May 4 15:43:40 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Sat May 4 15:43:40 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Sat May 4 15:43:40 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "resume-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha/conf/dm-task.yaml" dmctl test cmd: "resume-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_basic/conf/dm-task.yaml" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/duplicate_event/source1.yaml" check diff successfully dmctl test cmd: "query-status test" check diff successfully dmctl test cmd: "purge-relay --filename dm-it-153b7508-05f7-475f-a541-8c005c6b14f2-9w8f8-6n44r-bin.000003 -s mysql-replica-01" wait process dm-master.test exit... wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:41 CST 2024] <<<<<< test case handle_error_2 success! >>>>>> start running case: [handle_error_3] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sat May 4 15:43:41 CST 2024] <<<<<< finish DM-097 optimistic >>>>>> got=2 expected=2 dmctl test cmd: "encrypt abc" encrypt value: Ph74sOAxCMyR1qYGa+lWJUAsL4w= 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/duplicate_event/conf/dm-task.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:41 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time use sync_diff_inspector to check increment2 data now! check diff successfully 4 dm-master alive 3 dm-worker alive 0 dm-syncer alive wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "show-ddl-locks" [Sat May 4 15:43:42 CST 2024] <<<<<< start DM-098 pessimistic >>>>>> got=1 expected=1 run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sat May 4 15:43:42 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 2 wait for rpc addr 127.0.0.1:8261 alive the 2-th time check log contain failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source1.yaml" wait process dm-master.test exit... process dm-master.test already exit [Sat May 4 15:43:43 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Sat May 4 15:43:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "show-ddl-locks" wait process dm-worker.test exit... got=1 expected=1 dmctl test cmd: "stop-task test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/handle_error_3/source2.yaml" wait process dm-master.test exit... [Sat May 4 15:43:44 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... check log contain failed 2-th time, retry later [Sat May 4 15:43:45 CST 2024] <<<<<< finish DM-UpdateBARule optimistic >>>>>> [Sat May 4 15:43:45 CST 2024] <<<<<< start DM-RENAME_TABLE pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-worker1 exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source-only-ca.yaml" wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait process dm-master.test exit... [Sat May 4 15:43:46 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-worker1 exit... process dm-worker1 already exit [Sat May 4 15:43:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/dmctl_basic/master/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... check diff failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:43:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" check log contain failed 3-th time, retry later got=2 expected=2 wait process dm-worker.test exit... [Sat May 4 15:43:47 CST 2024] <<<<<< finish DM-098 pessimistic >>>>>> got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: full mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:1559 alter table handle_error.tb1 add column d varchar(20);" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait process dm-master.test exit... { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task-2.yaml --remove-meta=true" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:1735 alter table handle_error.tb1 add column d varchar(20);" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit clean source table dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=3 expected=3 run tidb sql failed 1-th time, retry later check diff successfully dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchAllDoTablesFailed=return(1152) check log contain failed 4-th time, retry later [Sat May 4 15:43:49 CST 2024] <<<<<< finish DM-RENAME_TABLE pessimistic >>>>>> [Sat May 4 15:43:49 CST 2024] <<<<<< start DM-RENAME_TABLE optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" { "result": true, "msg": "", "sources": [ { "result": true, "msg": "", "source": "mysql-replica-01", "worker": "worker1" } ], "checkResult": "pre-check is passed. " } PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/dm/... dmctl test cmd: "query-status test" dmctl test cmd: "query-status test2" wait process dm-master.test exit... [Sat May 4 15:43:49 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "encrypt a" 1 dm-master alive 0 dm-worker alive 0 dm-syncer alive check data check diff successfully check diff successfully ============================== test_worker_handle_multi_tls_tasks success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-worker.test exit... import prepare data start DM worker and master cluster [Sat May 4 15:43:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master1.toml >>>>>> [Sat May 4 15:43:50 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master2.toml >>>>>> [Sat May 4 15:43:50 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit dmctl test cmd: "stop-task test" [Sat May 4 15:43:51 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Sat May 4 15:43:51 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 5-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "stop-task test" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:51 CST 2024] <<<<<< test case dmctl_basic success! >>>>>> start running case: [dmctl_command] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/run.sh... Verbose mode = false wait process dm-worker.test exit... rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive start worker and operate mysql config to worker [Sat May 4 15:43:51 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time [Sat May 4 15:43:52 CST 2024] <<<<<< finish DM-RENAME_TABLE optimistic >>>>>> [Sat May 4 15:43:52 CST 2024] <<<<<< start DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" check un-accessible DM-worker exists dmctl test cmd: "query-status -s 127.0.0.1:8888" wait process dm-master.test exit... start task and will failed dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml" reset go failpoints, and need restart dm-worker, then start task again wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:43:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-worker.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source1.yaml" wait process dm-worker.test exit... wait process dm-master.test exit... dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:3856 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:4247 alter table handle_error.tb1 add column d varchar(20);" check log contain failed 6-th time, retry later dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 2-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" got=2 expected=2 dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 got=3 expected=3 wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:53 CST 2024] <<<<<< test case shardddl3 success! >>>>>> start running case: [shardddl3_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:43:53 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-master.toml >>>>>> dmctl test cmd: "stop-task test" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:43:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sat May 4 15:43:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive exit code should be not zero exit code should be not zero [Sat May 4 15:43:54 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> [Sat May 4 15:43:54 CST 2024] <<<<<< start DM-4189 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" [Sat May 4 15:43:54 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" [Sat May 4 15:43:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:55 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "pause-task test" wait process dm-master.test exit... wait process dm-master.test exit... check log contain failed 7-th time, retry later dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:43:55 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases_2/source2.yaml" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" got=2 expected=2 wait process dm-master.test exit... wait process dm-master.test exit... dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:6411 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source1.yaml" start DM task dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-task.yaml " got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:6759 alter table handle_error.tb1 add column d varchar(20);" dmctl test cmd: "query-status test" wait process dm-master.test exit... process dm-master.test already exit [Sat May 4 15:43:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/source1-encrypted-pass.yaml" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" dmctl test cmd: "query-status test" wait process dm-worker.test exit... dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/wrong-source.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml -w worker1" wait process dm-master.test exit... got=3 expected=3 dmctl test cmd: "stop-task test" wait process dm-master.test exit... check diff successfully dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "shard-ddl-lock" dmctl test cmd: "list-member --name worker1" dmctl test cmd: "validation status test" [Sat May 4 15:43:57 CST 2024] <<<<<< finish DM-4189 pessimistic >>>>>> [Sat May 4 15:43:57 CST 2024] <<<<<< start DM-4189 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" [Sat May 4 15:43:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w wrong-worker" dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml -w worker1" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again [Sat May 4 15:43:57 CST 2024] <<<<<< finish DM-RENAME_COLUMN_OPTIMISTIC optimistic >>>>>> dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" check log contain failed 8-th time, retry later dmctl test cmd: "query-status test" wait process dm-master.test exit... got=2 expected=2 got=2 expected=2 use sync_diff_inspector to check full dump loader check diff successfully flush logs to force rotate binlog file apply increment data before restart dm-worker to ensure entering increment phase wait process dm-worker.test exit... process dm-worker.test already exit success success [Sat May 4 15:43:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> [Sat May 4 15:43:57 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/shardddl3_1/source2.yaml" wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 rpc addr 127.0.0.1:8262 is alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "resume-task test" dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /tmp/dm_test/dmctl_command/wrong-dm-task.yaml" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml" dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:8966 alter table handle_error.tb1 add column d varchar(20);" wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit got=3 expected=3 wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:43:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:9271 alter table handle_error.tb1 add column d varchar(20);" check log contain failed 9-th time, retry later [Sat May 4 15:43:59 CST 2024] <<<<<< start DM-099 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c varchar(20);" wait process dm-master exit... process dm-master already exit [Sat May 4 15:43:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "stop-task test" got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check we can catch inconsistent rows: fast mode 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sat May 4 15:44:00 CST 2024] <<<<<< finish DM-4189 optimistic >>>>>> wait process tidb-server exit... process tidb-server already exit mysql_ssl_setup at=/var/lib/mysql/ add dm_tls_test user done /var/lib/mysql/ run a new tidb server with tls wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive check diff successfully wait process dm-master.test exit... check diff successfully dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again dmctl test cmd: "query-status test" got=2 expected=2 wait process dm-master.test exit... check diff successfully wait for rpc addr 127.0.0.1:8261 alive the 2-th time wait process dm-master.test exit... process dm-master.test already exit [Sat May 4 15:44:01 CST 2024] <<<<<< start DM-4210 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" check log contain failed 10-th time, retry later wait process dm-master.test exit... wait process dm-worker2 exit... use sync_diff_inspector to check increment data check diff successfully [Sat May 4 15:44:01 CST 2024] <<<<<< finish test_running >>>>>> start dumping SQLs into source kill dm-worker1 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" got=3 expected=3 check sync diff after clean SafeModeExit failpoint check diff successfully check sync diff after restart DDL owner rpc addr 127.0.0.1:8261 is alive dmctl test cmd: "list-member -w" got=2 expected=2 [Sat May 4 15:44:02 CST 2024] <<<<<< start DM-RemoveLock pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... dmctl test cmd: "binlog replace test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:11239 alter table handle_error.tb1 add column d int;alter table handle_error.tb1 add unique(d);" wait process dm-master.test exit... wait process dm-worker2 exit... wait process dm-worker1 exit... got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 1-th time, will retry again dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column e int unique;" dmctl test cmd: "query-status test" process dm-worker2 already exit [Sat May 4 15:44:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;alter table handle_error.tb1 add unique(c);" dmctl test cmd: "query-status test" got=2 expected=2 run tidb sql failed 1-th time, retry later wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 check log contain failed 1-th time, retry later check log contain failed 11-th time, retry later wait process dm-worker1 exit... process dm-worker1 already exit start worker3 [Sat May 4 15:44:03 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker3.toml >>>>>> wait for rpc addr 127.0.0.1:8264 alive the 1-th time rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:44:04 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sat May 4 15:44:04 CST 2024] <<<<<< finish DM-099 pessimistic >>>>>> rpc addr 127.0.0.1:8264 is alive dmctl test cmd: "start-relay -s mysql-replica-01 worker3" got=1 expected=1 got=1 expected=1 got=2 expected=2 got=2 expected=2 dmctl test cmd: "query-status test" got=1 expected=1 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... [Sat May 4 15:44:05 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/tls/dm-master1.toml >>>>>> [Sat May 4 15:44:05 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /tmp/dm_test/tls/dm-master2.toml >>>>>> [Sat May 4 15:44:05 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /tmp/dm_test/tls/dm-master3.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml " dmctl test cmd: "stop-task test" [Sat May 4 15:44:05 CST 2024] <<<<<< start DM-100 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:44:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> check diff successfully dmctl test cmd: "pause-task test" dmctl test cmd: "query-status test" got=2 expected=2 success check log contain failed 12-th time, retry later wait for rpc addr 127.0.0.1:8262 alive the 1-th time success finish running run safe mode recover case 2 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive [Sat May 4 15:44:06 CST 2024] <<<<<< finish DM-4210 >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... got=2 expected=2 kill dm-worker2 check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml" wait process dm-master.test exit... check diff successfully 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive dmctl test cmd: "query-status test" got=2 expected=2 [Sat May 4 15:44:07 CST 2024] <<<<<< start DM-4193 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-pessimistic.yaml --remove-meta" [Sat May 4 15:44:07 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-master.test exit... wait process dm-worker2 exit... check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 107 100 107 0 0 94024 0 --:--:-- --:--:-- --:--:-- 104k wait process dm-master.test exit... process dm-master.test already exit check log contain failed 13-th time, retry later wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... wait for rpc addr 127.0.0.1:8261 alive the 2-th time rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit start worker4 [Sat May 4 15:44:08 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_cases_2/conf/dm-worker4.toml >>>>>> wait process dm-worker1 exit... check diff successfully dmctl test cmd: "stop-task test" wait for rpc addr 127.0.0.1:18262 alive the 1-th time wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" [Sat May 4 15:44:08 CST 2024] <<<<<< finish DM-RemoveLock pessimistic >>>>>> [Sat May 4 15:44:08 CST 2024] <<<<<< start DM-RemoveLock optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta" got=2 expected=2 wait process dm-master.test exit... process dm-master.test already exit rpc addr 127.0.0.1:8261 is alive rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive [Sat May 4 15:44:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/tls/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:13027" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:12152" wait process dm-worker.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:13027" rpc addr 127.0.0.1:18262 is alive wait and check task running wait process dm-worker.test exit... dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again wait process dm-master.test exit... dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:12152" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:13413" HTTP 127.0.0.1:8261/apis/v1alpha1/status/test is alive query-status from all dm-master dmctl test cmd: "query-status test" got=3 expected=3 dmctl test cmd: "query-status test" wait process dm-worker1 exit... process dm-worker1 already exit [Sat May 4 15:44:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time use sync_diff_inspector to check increment data check diff successfully [Sat May 4 15:44:03 CST 2024] <<<<<< finish test_isolate_master_and_worker >>>>>> 5 dm-master alive 5 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... wait process dm-master.test exit... check log contain failed 14-th time, retry later dmctl test cmd: "binlog revert test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:13413" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:13413" wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "query-status test" % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 121 100 121 0 0 86428 0 --:--:-- --:--:-- --:--:-- 118k dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:12590" dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker.test exit... "is_initialized": true, Starting TiDB... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused check log contain failed 1-th time, retry later rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/tls/source1.yaml" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive check diff successfully dmctl test cmd: "stop-task test" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" [Sat May 4 15:44:10 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-master.toml >>>>>> wait process dm-worker.test exit... got=3 expected=3 dmctl test cmd: "stop-task test" [Sat May 4 15:44:10 CST 2024] <<<<<< finish DM-4193 pessimistic >>>>>> [Sat May 4 15:44:10 CST 2024] <<<<<< start DM-4193 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/double-source-optimistic.yaml --remove-meta" wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit start to run safe mode case 3 [Sat May 4 15:44:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time [Sat May 4 15:44:10 CST 2024] <<<<<< finish DM-100 optimistic >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... check log contain failed 15-th time, retry later wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:44:11 CST 2024] <<<<<< start test_COMMIT_in_QueryEvent >>>>>> 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit start task and check stage dmctl test cmd: "start-task /tmp/dm_test/tls/dm-task.yaml --remove-meta=true" [Sat May 4 15:44:11 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/start_task/1/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check update pk(split into insert and delete) [Sat May 4 15:44:11 CST 2024] <<<<<< start DM-101 optimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta" check log contain failed 1-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:44:12 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "query-status test" wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master.test exit... process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:44:12 CST 2024] <<<<<< test case ha_cases3_1 success! >>>>>> start running case: [ha_master] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/run.sh... Verbose mode = false 0 dm-master alive 0 dm-worker alive 0 dm-syncer alive process dm-master.test already exit process dm-worker.test already exit process dm-syncer.test already exit start DM worker and master [Sat May 4 15:44:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>> [Sat May 4 15:44:12 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>> [Sat May 4 15:44:12 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>> [Sat May 4 15:44:12 CST 2024] <<<<<< START DM-MASTER on port 8561, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master4.toml >>>>>> [Sat May 4 15:44:12 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time got=2 expected=2 dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:15189" dmctl test cmd: "binlog skip test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:14529" dmctl test cmd: "query-status test" wait process dm-master.test exit... dmctl test cmd: "query-status test" got=2 expected=2 dmctl test cmd: "binlog revert test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:15189" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:44:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/start_task/1/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "validation status test" got=0 expected=1 command: validation status test "processedRowsStatus": "insert\/update\/delete: 7\/1\/2" count: 0 != expected: 1, failed the 0-th time, will retry again check data check diff successfully pause task before kill and restart dm-worker dmctl test cmd: "pause-task test" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:44:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time dmctl test cmd: "query-status test" got=2 expected=2 check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:14529" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:15575" rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:44:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time rpc addr 127.0.0.1:8261 is alive start dm-worker2 and kill dm-worker1 dmctl test cmd: "binlog revert test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:15575" dmctl test cmd: "binlog skip test -s mysql-replica-01 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:15575" rpc addr 127.0.0.1:8361 is alive rpc addr 127.0.0.1:8461 is alive dmctl test cmd: "binlog revert test -s mysql-replica-02 -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:14967" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8561 is alive rpc addr 127.0.0.1:8661 is alive check log contain failed 16-th time, retry later wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml" got=2 expected=2 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:44:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=2 expected=2 got=1 expected=1 check master alive dmctl test cmd: "list-member" got=1 expected=1 start task and check stage dmctl test cmd: "start-task /tmp/dm_test/start_task/1/dm-task.yaml --remove-meta=true" got=1 expected=1 dmctl test cmd: "binlog skip test" dmctl test cmd: "query-status test" got=3 expected=3 wait process dm-worker1 exit... check diff successfully dmctl test cmd: "stop-task test" wait process dm-worker2 exit... dmctl test cmd: "stop-task test" [Sat May 4 15:44:14 CST 2024] <<<<<< finish DM-4193 optimistic >>>>>> rpc addr 127.0.0.1:8263 is alive wait process dm-master.test exit... wait process dm-worker2 exit... process dm-worker2 already exit [Sat May 4 15:44:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time got=1 expected=1 got=0 expected=1 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml" dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml" wait process dm-worker1 exit... process dm-worker1 already exit mysql_ssl_setup at=/var/lib/mysql/ [Sat May 4 15:44:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /tmp/dm_test/tls/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time [Sat May 4 15:44:15 CST 2024] <<<<<< start DM-4230 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" check log contain failed 17-th time, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:10080; Connection refused dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source1.yaml" wait process dm-master.test exit... got=2 expected=2 dmctl test cmd: "query-status test" got=2 expected=2 got=1 expected=1 got=2 expected=2 check data dmctl test cmd: "operate-source create /tmp/dm_test/dmctl_command/source2.yaml" check diff successfully <<<<<< test_COMMIT_in_QueryEvent success! >>>>>> 1 dm-master alive 1 dm-worker alive 0 dm-syncer alive rpc addr 127.0.0.1:8263 is alive dmctl test cmd: "query-status test" rpc addr 127.0.0.1:8263 is alive check diff successfully dmctl test cmd: "stop-task test" resume task to worker2 dmctl test cmd: "resume-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta" [Sat May 4 15:44:17 CST 2024] <<<<<< finish DM-101 optimistic >>>>>> dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" dmctl test cmd: "query-status test" wait process dm-master.test exit... got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column d int unique;" dmctl test cmd: "query-status test" check diff successfully ============================== test_worker_ha_when_enable_source_tls success ================================== 3 dm-master alive 1 dm-worker alive 0 dm-syncer alive wait process dm-master.test exit... got=1 expected=1 got=1 expected=1 got=1 expected=1 --> check validator panic and we can catch it 1 dm-master alive 2 dm-worker alive 0 dm-syncer alive got=1 expected=1 dmctl test cmd: "binlog revert test" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int;" check log contain failed 18-th time, retry later dmctl test cmd: "query-status test" got=2 expected=2 [Sat May 4 15:44:18 CST 2024] <<<<<< start DM-102 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" wait process dm-master.test exit... dmctl test cmd: "stop-task test" dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task2.yaml --remove-meta" wait process dm-master.test exit... check diff successfully [Sat May 4 15:44:18 CST 2024] <<<<<< finish DM-4230 >>>>>> wait process dm-master.test exit... wait process dm-master.test exit... process dm-master.test already exit wait process dm-master.test exit... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 123 100 123 0 0 192k 0 --:--:-- --:--:-- --:--:-- 120k {"connections":0,"version":"8.0.11-TiDB-v8.2.0-alpha-79-g600b2ed4bf","git_hash":"600b2ed4bf0aa38224a1c4c4c68831820735515c"}dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/many_tables/conf/dm-task-2.yaml " kill dm-master1 and kill dm-master2 dmctl test cmd: "query-status test" got=2 expected=2 [Sat May 4 15:44:19 CST 2024] <<<<<< start DM-4177 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master.test exit... process dm-master.test already exit check log contain failed 19-th time, retry later wait process dm-master.test exit... wait process dm-worker.test exit... wait process dm-worker.test exit... wait process dm-master.test exit... wait process dm-master1 exit... dmctl test cmd: "query-status test2" dmctl test cmd: "stop-task test2" wait process dm-worker.test exit... dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/dmctl_command/conf/dm-task.yaml --remove-meta" wait process dm-master.test exit... wait process dm-worker.test exit... process dm-worker.test already exit [Sat May 4 15:44:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time dmctl test cmd: "shard-ddl-lock" got=1 expected=1 dmctl test cmd: "shard-ddl-lock unlock test-`shardddl`.`tb`" dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 wait process dm-worker.test exit... run tidb sql failed 1-th time, retry later dmctl test cmd: "binlog skip wrong-task" dmctl test cmd: "binlog skip" wait process dm-master1 exit... process dm-master1 already exit dmctl test cmd: "binlog skip test -b dm-it-4f1125b6-631d-42e7-8341-528c6efa3403-f3zn5-9hlr2-bin|000001.000001:19397 -s wrong-source" dmctl test cmd: "binlog skip test -b mysql-bin|1111 -s wrong-source" wait process dm-master.test exit... check log contain failed 20-th time, retry later dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" wait process dm-master.test exit... rpc addr 127.0.0.1:8262 is alive [Sat May 4 15:44:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>> wait for rpc addr 127.0.0.1:8263 alive the 1-th time wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:44:21 CST 2024] <<<<<< test case start_task success! >>>>>> start running case: [print_status] script: [/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh] Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh... Verbose mode = false +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status ++ pwd + cur=/home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/../_utils/test_prepare ++ source env_variables +++ MYSQL_HOST1=127.0.0.1 +++ MYSQL_HOST2=127.0.0.1 +++ TIDB_HOST=127.0.0.1 +++ MYSQL_PORT1=3306 +++ MYSQL_PORT2=3307 +++ MYSQL_PASSWORD1=123456 +++ MYSQL_PASSWORD2=123456 +++ TIDB_PASSWORD=123456 +++ TIDB_PORT=4000 +++ MASTER_PORT=8261 +++ MASTER_PEER_PORT=8291 +++ MASTER_PORT1=8261 +++ MASTER_PORT2=8361 +++ MASTER_PORT3=8461 +++ MASTER_PORT4=8561 +++ MASTER_PORT5=8661 +++ MASTER_PORT6=8761 +++ MASTER_PEER_PORT1=8291 +++ MASTER_PEER_PORT2=8292 +++ MASTER_PEER_PORT3=8293 +++ MASTER_PEER_PORT4=8294 +++ MASTER_PEER_PORT5=8295 +++ MASTER_PEER_PORT6=8296 +++ WORKER1_PORT=8262 +++ WORKER2_PORT=8263 +++ WORKER3_PORT=8264 +++ WORKER4_PORT=18262 +++ WORKER5_PORT=18263 +++ WORKER1_NAME=worker1 +++ WORKER2_NAME=worker2 +++ SOURCE_ID1=mysql-replica-01 +++ SOURCE_ID2=mysql-replica-02 +++ RESET_MASTER=true +++ VERBOSE=false ++ '[' true = true ']' ++ run_sql 'RESET MASTER' 3306 123456 ++ run_sql 'RESET MASTER' 3307 123456 wait process dm-worker.test exit... check diff successfully dmctl test cmd: "validation status test" dmctl test cmd: "binlog skip test" dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int;"" + WORK_DIR=/tmp/dm_test/print_status + cleanup_data print_status + rm -rf /tmp/dm_test/print_status + mkdir /tmp/dm_test/print_status + for target_db in '"$@"' + run_sql 'drop database if exists `print_status`' 4000 123456 + run_sql 'drop database if exists dm_meta' 4000 123456 + cleanup_process ++ ps aux ++ grep dm-master.test temp ++ wc -l ++ rm temp + dm_master_num=0 + echo '0 dm-master alive' 0 dm-master alive + pkill -hup dm-master.test + true ++ ps aux ++ grep dm-worker.test temp ++ wc -l ++ rm temp + dm_worker_num=0 + echo '0 dm-worker alive' 0 dm-worker alive + pkill -hup dm-worker.test + true ++ ps aux ++ grep dm-syncer.test temp ++ wc -l ++ rm temp + dm_syncer_num=0 + echo '0 dm-syncer alive' 0 dm-syncer alive + pkill -hup dm-syncer.test + true + wait_process_exit dm-master.test process dm-master.test already exit + wait_process_exit dm-worker.test process dm-worker.test already exit + wait_process_exit dm-syncer.test process dm-syncer.test already exit + run + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.prepare.sql 127.0.0.1 3306 123456 + inject_points=("github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)" "github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)" "github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)") ++ join_string ';' 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)' 'github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)' 'github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' ++ local 'IFS=;' ++ shift ++ echo 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + GO_FAILPOINTS='github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)' + cp /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-worker1.toml /tmp/dm_test/print_status/dm-worker1.toml + sed -i s%placeholder%/tmp/dm_test/print_status/relay_by_worker%g /tmp/dm_test/print_status/dm-worker1.toml + run_dm_master /tmp/dm_test/print_status/master 8261 /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml [Sat May 4 15:44:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_master_online 127.0.0.1:8261 wait for rpc addr 127.0.0.1:8261 alive the 1-th time wait process dm-master2 exit... got=1 expected=1 got=1 expected=1 got=0 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again dmctl test cmd: "binlog skip test -b mysql-bin.000000:00000" dmctl test cmd: "binlog aaa test" got=1 expected=1 dmctl test cmd: "stop-task test" [Sat May 4 15:44:22 CST 2024] <<<<<< finish DM-102 >>>>>> run tidb sql failed 1-th time, retry later wait process dm-master.test exit... wait process dm-master.test exit... rpc addr 127.0.0.1:8263 is alive wait process dm-worker.test exit... process dm-worker.test already exit process dm-syncer.test already exit [Sat May 4 15:44:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>> wait for rpc addr 127.0.0.1:8261 alive the 1-th time Aborted by Jenkins Admin dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "pause-task test" run tidb sql failed 2-th time, retry later dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t1 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-01 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" Sending interrupt signal to process Killing processes wait process dm-master.test exit... dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t2 /tmp/dm_test/safe_mode/schema.sql" dmctl test cmd: "operate-schema set -s mysql-replica-02 test -d safe_mode_test -t t3 /tmp/dm_test/safe_mode/schema.sql" rpc addr 127.0.0.1:8261 is alive + run_dm_worker /tmp/dm_test/print_status/worker1 8262 /tmp/dm_test/print_status/dm-worker1.toml [Sat May 4 15:44:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/print_status/dm-worker1.toml >>>>>> + check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_worker_online 127.0.0.1:8262 wait for rpc addr 127.0.0.1:8262 alive the 1-th time wait process dm-master2 exit... kill finished with exit code 0 Sending interrupt signal to process Killing processes wait process dm-master.test exit... make: *** [dm_integration_test_in_group] Terminated [Sat May 4 15:44:23 CST 2024] <<<<<< start DM-103 pessimistic >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta" script returned exit code 143 rpc addr 127.0.0.1:8261 is alive [Sat May 4 15:44:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>> wait for rpc addr 127.0.0.1:8262 alive the 1-th time kill finished with exit code 0 Sending interrupt signal to process Killing processes restore config make: *** [dm_integration_test_in_group] Terminated /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/validator_basic/run.sh: line 53: 3887 Terminated check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT restore time_zone ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes check log contain failed 21-th time, retry later dmctl test cmd: "stop-task test" wait process dm-master.test exit... kill finished with exit code 0 Sending interrupt signal to process Killing processes rpc addr 127.0.0.1:8262 is alive + cp /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/source1.yaml /tmp/dm_test/print_status/source1.yaml + sed -i '/relay-binlog-name/i\relay-dir: /tmp/dm_test/print_status/worker1/relay_log' /tmp/dm_test/print_status/source1.yaml + dmctl_operate_source create /tmp/dm_test/print_status/source1.yaml mysql-replica-01 + operate=create + source_toml=/tmp/dm_test/print_status/source1.yaml + source_id=mysql-replica-01 + run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'operate-source create /tmp/dm_test/print_status/source1.yaml' '"result": true' 2 '"source": "mysql-replica-01"' 1 make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 wait process dm-master2 exit... [Sat May 4 15:44:24 CST 2024] <<<<<< finish DM-4177 >>>>>> dmctl test cmd: "binlog skip test" make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 got=1 expected=1 got=1 expected=1 got=1 expected=2 command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 1-th time, will retry again wait process dm-worker.test exit... dmctl test cmd: "binlog replace test alter table handle_error.tb1 add column c int" dmctl test cmd: "binlog revert test" run tidb sql failed 3-th time, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes wait process dm-master2 exit... kill finished with exit code 0 Sending interrupt signal to process Killing processes dmctl test cmd: "query-status test" make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 wait process dm-worker.test exit... process dm-worker.test already exit check log contain failed 22-th time, retry later [Sat May 4 15:44:25 CST 2024] <<<<<< start DM-4231 >>>>>> dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/pull_dm_integration_test/tiflow/dm/tests/handle_error_3/conf/single-source-no-sharding.yaml --remove-meta" wait process dm-master2 exit... got=1 expected=1 got=1 expected=1 got=2 expected=2 got=1 expected=1 got=1 expected=1 got=4 expected=4 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 got=1 expected=1 got=1 expected=1 got=1 expected=1 make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes run tidb sql failed 4-th time, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 dmctl test cmd: "query-status test" wait process dm-master2 exit... kill finished with exit code 0 Sending interrupt signal to process Killing processes make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 got=1 expected=1 dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog replace test "alter table handle_error.tb1 add column c int; alter table handle_error.tb1 add column d int unique;"" kill finished with exit code 0 Sending interrupt signal to process Killing processes dmctl test cmd: "query-status test" got=1 expected=1 dmctl test cmd: "binlog skip test" kill finished with exit code 0 Sending interrupt signal to process Killing processes make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 dmctl test cmd: "stop-task test" make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes wait process dm-master2 exit... process dm-master2 already exit make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 kill finished with exit code 0 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } make: *** [dm_integration_test_in_group] Terminated script returned exit code 143 [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // stage [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } Click here to forcibly terminate running steps [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // container [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // node [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G01' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G03' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G06' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G07' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'TLS_GROUP' [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G00' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: 74c78f0d-d065-4ba9-bd6a-d245c7a910a0 Failed in branch Matrix - TEST_GROUP = 'G01' Finished: ABORTED