Skipping 1,171 KB.. Full Log

wait for rpc addr 127.0.0.1:8262 alive the 1-th time
got=2 expected=2
check diff successfully
check diff successfully
dmctl test cmd: "stop-task test"
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:35:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
[Sun Apr 28 15:35:20 CST 2024] <<<<<< finish DM-131 optimistic >>>>>>
[Sun Apr 28 15:35:21 CST 2024] <<<<<< start DM-132 pessimistic >>>>>>
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test validator panic count: 0 != expected: 1, failed the 0-th time, will retry again
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:35:24 CST 2024] <<<<<< finish test_query_timeout >>>>>>
[Sun Apr 28 15:35:24 CST 2024] <<<<<< start test_regexpr_router regexpr-task.yaml >>>>>>
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
got=1 expected=1
--> check validator worker panic and we can catch it
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 15:35:25 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-master.test exit...
process dm-master.test already exit
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:35:25 CST 2024] <<<<<< finish DM-132 pessimistic >>>>>>
[Sun Apr 28 15:35:25 CST 2024] <<<<<< start DM-132 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:35:27 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:35:27 CST 2024] <<<<<< finish DM-132 optimistic >>>>>>
wait process dm-worker.test exit...
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 15:35:28 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:35:28 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
[Sun Apr 28 15:35:28 CST 2024] <<<<<< start DM-133 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml"
dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml"
dmctl test cmd: "query-status test"
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
got=2 expected=2
dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta"
dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task.yaml --remove-meta"
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:35:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
check diff successfully
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 15:35:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-master.test exit...
process dm-master.test already exit
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:35:33 CST 2024] <<<<<< finish DM-133 pessimistic >>>>>>
[Sun Apr 28 15:35:33 CST 2024] <<<<<< start DM-133 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
check diff successfully
dmctl test cmd: "stop-task test"
wait process dm-worker.test exit...
[Sun Apr 28 15:35:36 CST 2024] <<<<<< finish DM-133 optimistic >>>>>>
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:35:37 CST 2024] <<<<<< start DM-134 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
[Sun Apr 28 15:35:38 CST 2024] <<<<<< finish test_regexpr_router regexpr-task.yaml >>>>>>
[Sun Apr 28 15:35:38 CST 2024] <<<<<< start test_regexpr_router regexpr-task-lightning.yaml >>>>>>
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:35:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:35:39 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "validation status test"
got=1 expected=1
dmctl test cmd: "validation start test"
dmctl test cmd: "validation status test"
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 0-th time, will retry again
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 15:35:40 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:35:41 CST 2024] <<<<<< finish DM-134 pessimistic >>>>>>
[Sun Apr 28 15:35:41 CST 2024] <<<<<< start DM-134 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml"
dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml"
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> check validator stop when pending row size too large
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
dmctl test cmd: "query-status test"
got=2 expected=2
check diff successfully
dmctl test cmd: "stop-task test"
dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta"
dmctl test cmd: "start-task /tmp/dm_test/all_mode/regexpr-task-lightning.yaml --remove-meta"
[Sun Apr 28 15:35:43 CST 2024] <<<<<< finish DM-134 optimistic >>>>>>
wait process dm-master.test exit...
process dm-master.test already exit
check diff successfully
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 15:35:44 CST 2024] <<<<<< start DM-135 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-worker.test exit...
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:35:46 CST 2024] <<<<<< finish DM-135 pessimistic >>>>>>
[Sun Apr 28 15:35:46 CST 2024] <<<<<< start DM-135 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:35:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-worker.test exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:35:47 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:35:48 CST 2024] <<<<<< finish DM-135 optimistic >>>>>>
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:35:48 CST 2024] <<<<<< finish test_regexpr_router regexpr-task-lightning.yaml >>>>>>
[Sun Apr 28 15:35:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:35:48 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
[Sun Apr 28 15:35:49 CST 2024] <<<<<< start DM-136 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:35:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=2 expected=2
check diff successfully
dmctl test cmd: "stop-task test"
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml"
[Sun Apr 28 15:35:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
[Sun Apr 28 15:35:51 CST 2024] <<<<<< finish DM-136 optimistic >>>>>>
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source2.yaml"
[Sun Apr 28 15:35:52 CST 2024] <<<<<< start DM-137 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again
dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=2 expected=2
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:35:54 CST 2024] <<<<<< finish DM-137 optimistic >>>>>>
dmctl test cmd: "query-status t-Ë!s`t"
got=1 expected=1
dmctl test cmd: "stop-task /tmp/dm_test/all_mode/dm-task.yaml"
dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml --remove-meta"
got=0 expected=1
command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again
[Sun Apr 28 15:35:55 CST 2024] <<<<<< start DM-138 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
check diff successfully
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
dmctl test cmd: "query-status -s mysql-replica-01"
dmctl test cmd: "pause-task t-Ë!s`t"
dmctl test cmd: "operate-schema set -s mysql-replica-01 t-Ë!s`t -d all_mode -t no_diff /tmp/dm_test/all_mode/schema.sql"
got=1 expected=1
got=1 expected=1+
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> check validator stop when pending row count too many
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
dmctl test cmd: "resume-task t-Ë!s`t"
wait for dm to sync
use sync_diff_inspector to check data now!
check diff successfully
[Sun Apr 28 15:35:57 CST 2024] <<<<<< finish test_kill_worker_in_sync >>>>>>
3 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait pattern dm-worker1.toml exit...
wait process dm-master.test exit...
process dm-master.test already exit
wait pattern dm-worker1.toml exit...
pattern dm-worker1.toml already exit
[Sun Apr 28 15:35:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-master.test exit...
check diff successfully
dmctl test cmd: "stop-task test"
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "query-status t-Ë!s`t"
wait process dm-worker.test exit...
got=1 expected=1
[Sun Apr 28 15:35:59 CST 2024] <<<<<< finish DM-138 pessimistic >>>>>>
[Sun Apr 28 15:35:59 CST 2024] <<<<<< start DM-138 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
wait pattern dm-worker2.toml exit...
wait pattern dm-worker2.toml exit...
pattern dm-worker2.toml already exit
[Sun Apr 28 15:35:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/all_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-master.test exit...
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
rpc addr 127.0.0.1:8263 is alive
wait process dm-master.test exit...
check diff successfully
dmctl test cmd: "stop-task test"
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:36:01 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
[Sun Apr 28 15:36:01 CST 2024] <<<<<< finish DM-138 optimistic >>>>>>
wait process dm-master.test exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:36:02 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
[Sun Apr 28 15:36:02 CST 2024] <<<<<< start DM-139 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-master.test exit...
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:36:03 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
dmctl test cmd: "query-status test"
wait process dm-master.test exit...
got=2 expected=2
check diff failed 1-th time, retry later
wait process dm-master.test exit...
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
wait process dm-master.test exit...
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:36:05 CST 2024] <<<<<< test case ha_cases_2 success! >>>>>>
start running case: [ha_cases2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh...
Verbose mode = false
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:36:05 CST 2024] <<<<<< start test_pause_task >>>>>>
[Sun Apr 28 15:36:05 CST 2024] <<<<<< start test_multi_task_running >>>>>>
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
clean source table
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:06 CST 2024] <<<<<< finish DM-139 pessimistic >>>>>>
[Sun Apr 28 15:36:06 CST 2024] <<<<<< start DM-139 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again
dmctl test cmd: "query-status test"
got=2 expected=2
check diff successfully
dmctl test cmd: "stop-task test"
import prepare data
start DM worker and master
[Sun Apr 28 15:36:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>>
[Sun Apr 28 15:36:08 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>>
[Sun Apr 28 15:36:08 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
[Sun Apr 28 15:36:08 CST 2024] <<<<<< finish DM-139 optimistic >>>>>>
rpc addr 127.0.0.1:8261 is alive
rpc addr 127.0.0.1:8361 is alive
rpc addr 127.0.0.1:8461 is alive
operate mysql config to worker
[Sun Apr 28 15:36:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
[Sun Apr 28 15:36:09 CST 2024] <<<<<< start DM-142 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
got=0 expected=1
command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 1-th time, will retry again
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml"
dmctl test cmd: "query-status test"
after restart dm-worker, task should resume automatically
dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task.yaml"
got=2 expected=2
check diff failed 1-th time, retry later
[Sun Apr 28 15:36:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
got=1 expected=1
got=1 expected=1+
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> start validator on the fly, validate from current syncer progress(gtid=false, relay=false)
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml"
check diff successfully
dmctl test cmd: "stop-task test"
HTTP 127.0.0.1:8261/apis/v1alpha1/status/t-Ë!s`t is alive
[Sun Apr 28 15:36:13 CST 2024] <<<<<< finish DM-142 pessimistic >>>>>>
wait process dm-master.test exit...
process dm-master.test already exit
[Sun Apr 28 15:36:14 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>>
wait for rpc addr 127.0.0.1:8264 alive the 1-th time
[Sun Apr 28 15:36:14 CST 2024] <<<<<< start DM-143 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-worker.test exit...
rpc addr 127.0.0.1:8264 is alive
[Sun Apr 28 15:36:15 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>>
wait for rpc addr 127.0.0.1:18262 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
wait process dm-worker.test exit...
check diff successfully
dmctl test cmd: "stop-task test"
rpc addr 127.0.0.1:18262 is alive
[Sun Apr 28 15:36:16 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>>
wait for rpc addr 127.0.0.1:18263 alive the 1-th time
[Sun Apr 28 15:36:16 CST 2024] <<<<<< finish DM-143 pessimistic >>>>>>
wait process tidb-server exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:36:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:18263 is alive
start DM task
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml "
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml "
wait process tidb-server exit...
[Sun Apr 28 15:36:17 CST 2024] <<<<<< start DM-145 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:36:18 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process tidb-server exit...
dmctl test cmd: "query-status test"
got=2 expected=2
got=2 expected=2
dmctl test cmd: "query-status test2"
got=2 expected=2
got=2 expected=2
use sync_diff_inspector to check full dump loader
check diff successfully
check diff successfully
flush logs to force rotate binlog file
apply increment data before restart dm-worker to ensure entering increment phase
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:36:19 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process tidb-server exit...
wait process tidb-server exit...
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
wait process tidb-server exit...
dmctl test cmd: "config source mysql-replica-01"
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:21 CST 2024] <<<<<< finish DM-145 pessimistic >>>>>>
[Sun Apr 28 15:36:21 CST 2024] <<<<<< start DM-145 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
got=1 expected=1
got=1 expected=1
wait process tidb-server exit...
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
wait process tidb-server exit...
check diff failed 1-th time, retry later
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
dmctl test cmd: "validation start test"
dmctl test cmd: "validation status test"
got=1 expected=1
use sync_diff_inspector to check increment data
check diff successfully
check diff successfully
[Sun Apr 28 15:36:24 CST 2024] <<<<<< finish test_multi_task_running >>>>>>
dmctl test cmd: "start-relay -s mysql-replica-01 worker1"
wait process tidb-server exit...
wait process tidb-server exit...
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:25 CST 2024] <<<<<< finish DM-145 optimistic >>>>>>
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
got=2 expected=2
dmctl test cmd: "start-relay -s mysql-replica-02 worker2"
wait process tidb-server exit...
[Sun Apr 28 15:36:26 CST 2024] <<<<<< start DM-146 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
got=2 expected=2
start dumping SQLs into source
pause tasks test
dmctl test cmd: "pause-task test"
dmctl test cmd: "pause-task test"
wait process tidb-server exit...
dmctl test cmd: "query-status test"
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
pause tasks test2
dmctl test cmd: "pause-task test2"
dmctl test cmd: "pause-task test2"
wait process tidb-server exit...
dmctl test cmd: "query-status test2"
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:28 CST 2024] <<<<<< finish DM-146 pessimistic >>>>>>
[Sun Apr 28 15:36:28 CST 2024] <<<<<< start DM-146 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
wait process tidb-server exit...
resume tasks test
dmctl test cmd: "resume-task test"
got=3 expected=3
dmctl test cmd: "resume-task test"
got=3 expected=3
dmctl test cmd: "query-status test"
resume tasks test2
dmctl test cmd: "resume-task test2"
dmctl test cmd: "query-status test"
got=2 expected=2
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator on the fly, validate from current syncer progress(gtid=false, relay=true)
got=3 expected=3
dmctl test cmd: "resume-task test2"
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "stop-task test"
wait process tidb-server exit...
process tidb-server already exit
got=3 expected=3
dmctl test cmd: "query-status test2"
[Sun Apr 28 15:36:30 CST 2024] <<<<<< finish DM-146 optimistic >>>>>>
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 15:36:31 CST 2024] <<<<<< start DM-147 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-master.test exit...
process dm-master.test already exit
Starting TiDB on port 4000
Verifying TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl4_1/schema.sql -s mysql-replica-01"
dmctl test cmd: "binlog replace test "alter table shardddl1.tb1 drop column b""
got=2 expected=2
got=1 expected=1
check diff successfully
dmctl test cmd: "stop-task test"
VARIABLE_NAME	VARIABLE_VALUE	COMMENT
bootstrapped	True	Bootstrap flag. Do not delete.
tidb_server_version	146	Bootstrap version. Do not delete.
system_tz	Asia/Shanghai	TiDB Global System Timezone.
new_collation_enabled	True	If the new collations are enabled. Do not edit it.
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'test'@'%'
[Sun Apr 28 15:36:34 CST 2024] <<<<<< finish DM-147 optimistic >>>>>>
wait process dm-worker.test exit...
[Sun Apr 28 15:36:35 CST 2024] <<<<<< start DM-148 pessimistic >>>>>>
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:36:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:36:36 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:36:37 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:39 CST 2024] <<<<<< finish DM-148 pessimistic >>>>>>
[Sun Apr 28 15:36:39 CST 2024] <<<<<< start DM-148 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
dmctl test cmd: "config source mysql-replica-01"
got=1 expected=1
got=1 expected=1
check diff successfully
dmctl test cmd: "pause-relay -s mysql-replica-01"
dmctl test cmd: "resume-relay -s mysql-replica-01"
dmctl test cmd: "query-status test"
got=2 expected=2
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:41 CST 2024] <<<<<< finish DM-148 optimistic >>>>>>
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
dmctl test cmd: "validation start test"
dmctl test cmd: "validation status test"
got=1 expected=1
relay logs dm-it-92095e73-fe5e-4ae3-b148-228b2de8f794-q1416-w0vxg-bin.000002
relay.meta
check diff successfully
check dump files have been cleaned
ls: cannot access /tmp/dm_test/all_mode/worker2/dumped_data.t-Ë!s`t: No such file or directory
worker2 auto removed dump files
check no password in log
dmctl test cmd: "query-status t-Ë!s`t"
got=1 expected=1
dmctl test cmd: "stop-task t-Ë!s`t"
matched
matched
[Sun Apr 28 15:36:42 CST 2024] <<<<<< start test_source_and_target_with_empty_gtid >>>>>>
[Sun Apr 28 15:36:42 CST 2024] <<<<<< start DM-149 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
wait process dm-worker.test exit...
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
wait process dm-worker.test exit...
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:46 CST 2024] <<<<<< finish DM-149 pessimistic >>>>>>
[Sun Apr 28 15:36:46 CST 2024] <<<<<< start DM-149 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:36:46 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/all_mode/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
dmctl test cmd: "query-status test"
got=2 expected=2
check diff successfully
dmctl test cmd: "stop-task test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator on the fly, validate from current syncer progress(gtid=true, relay=false)
[Sun Apr 28 15:36:48 CST 2024] <<<<<< finish DM-149 optimistic >>>>>>
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:36:48 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/all_mode/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 15:36:49 CST 2024] <<<<<< start DM-150 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/all_mode/source1.yaml"
got=2 expected=2
got=1 expected=1
check master alive
dmctl test cmd: "list-member"
wait process dm-master.test exit...
process dm-master.test already exit
got=1 expected=1
gtid is empty
start task and check stage
dmctl test cmd: "start-task /tmp/dm_test/all_mode/dm-task-no-gtid.yaml --remove-meta=true"
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "show-ddl-locks"
got=1 expected=1
check diff failed 1-th time, retry later
got=2 expected=2
dmctl test cmd: "query-status test"
got=2 expected=2
got=1 expected=1
got=2 expected=2
check data
check diff successfully
ERROR 1146 (42S02) at line 1: Table 'all_mode.t2' doesn't exist
run tidb sql failed 1-th time, retry later
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:54 CST 2024] <<<<<< finish DM-150 pessimistic >>>>>>
[Sun Apr 28 15:36:54 CST 2024] <<<<<< start DM-150 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=1 expected=1
check log contain failed 1-th time, retry later
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=3 expected=3
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:36:56 CST 2024] <<<<<< finish DM-150 optimistic >>>>>>
[Sun Apr 28 15:36:57 CST 2024] <<<<<< start DM-151 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-pessimistic.yaml --remove-meta"
dmctl test cmd: "query-status test"
[Sun Apr 28 15:36:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
got=2 expected=2
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "show-ddl-locks"
got=1 expected=1
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:37:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:37:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:37:02 CST 2024] <<<<<< finish DM-151 pessimistic >>>>>>
[Sun Apr 28 15:37:02 CST 2024] <<<<<< start DM-151 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=3 expected=3
dmctl test cmd: "config source mysql-replica-01"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
dmctl test cmd: "validation start test"
dmctl test cmd: "validation status test"
got=1 expected=1
check diff failed 1-th time, retry later
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:37:09 CST 2024] <<<<<< finish DM-151 optimistic >>>>>>
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
[Sun Apr 28 15:37:10 CST 2024] <<<<<< start DM-152 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=2 expected=2
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator on the fly, validate from current syncer progress(gtid=true, relay=true)
check diff failed 1-th time, retry later
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
check diff successfully
check diff failed 1-th time, retry later
wait process dm-worker.test exit...
wait process dm-worker.test exit...
check diff successfully
check diff failed 1-th time, retry later
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:37:17 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
check diff successfully
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:37:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:37:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
check diff successfully
check diff failed 1-th time, retry later
dmctl test cmd: "config source mysql-replica-01"
got=1 expected=1
got=1 expected=1
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:37:25 CST 2024] <<<<<< finish DM-152 optimistic >>>>>>
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
dmctl test cmd: "validation start test"
dmctl test cmd: "validation status test"
got=1 expected=1
dmctl test cmd: "query-status test"
got=1 expected=1
[Sun Apr 28 15:37:26 CST 2024] <<<<<< start DM-153 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 2\/1\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "binlog-schema update test shardddl1 tb1 -s mysql-replica-01 --from-target"
dmctl test cmd: "binlog replace test "alter table shardddl1.tb1 drop column b""
got=2 expected=2
got=1 expected=1
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:37:28 CST 2024] <<<<<< finish DM-153 optimistic >>>>>>
[Sun Apr 28 15:37:29 CST 2024] <<<<<< start DM-154 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "binlog-schema update test shardddl1 tb1 -s mysql-replica-01 --from-source"
dmctl test cmd: "binlog skip test"
got=2 expected=2
got=1 expected=1
check diff successfully
dmctl test cmd: "stop-task test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator from time < min mysql binlog pos(gtid=false)
[Sun Apr 28 15:37:32 CST 2024] <<<<<< finish DM-154 optimistic >>>>>>
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 15:37:33 CST 2024] <<<<<< start DM-155 optimistic >>>>>>
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:37:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:37:38 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
restart master
restart dm-master
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:37:39 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-master exit...
process dm-master already exit
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
[Sun Apr 28 15:37:41 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
dmctl test cmd: "config source mysql-replica-01"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
rpc addr 127.0.0.1:8261 is alive
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test"
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator from time < min mysql binlog pos(gtid=true)
restart master
restart dm-master
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master exit...
process dm-master already exit
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
[Sun Apr 28 15:37:50 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:37:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:37:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:37:54 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
restart worker1
restart dm-worker1
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
wait process worker1 exit...
dmctl test cmd: "config source mysql-replica-01"
got=1 expected=1
got=1 expected=1
wait process worker1 exit...
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
wait process worker1 exit...
got=1 expected=1
wait process worker1 exit...
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test"
wait process worker1 exit...
wait process worker1 exit...
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator from time which is in range of mysql binlog(gtid=false)
wait process worker1 exit...
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process worker1 exit...
process worker1 already exit
[Sun Apr 28 15:38:03 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-master.test exit...
process dm-master.test already exit
rpc addr 127.0.0.1:8262 is alive
wait process dm-worker.test exit...
restart worker2
restart dm-worker2
wait process dm-worker.test exit...
wait process worker2 exit...
process worker2 already exit
[Sun Apr 28 15:38:06 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:38:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:38:08 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
restart worker2
restart dm-worker2
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:38:09 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process worker2 exit...
process worker2 already exit
[Sun Apr 28 15:38:10 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "config source mysql-replica-01"
got=1 expected=1
got=1 expected=1
restart worker2
restart dm-worker2
wait process worker2 exit...
process worker2 already exit
[Sun Apr 28 15:38:13 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
rpc addr 127.0.0.1:8263 is alive
restart worker2
restart dm-worker2
dmctl test cmd: "validation start --start-time '2024-04-28 09:38:16' test"
wait process worker2 exit...
process worker2 already exit
[Sun Apr 28 15:38:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
rpc addr 127.0.0.1:8263 is alive
restart worker2
restart dm-worker2
wait process worker2 exit...
process worker2 already exit
[Sun Apr 28 15:38:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
rpc addr 127.0.0.1:8263 is alive
use sync_diff_inspector to check increment data
check diff successfully
check diff successfully
[Sun Apr 28 15:38:13 CST 2024] <<<<<< finish test_pause_task >>>>>>
[Sun Apr 28 15:38:13 CST 2024] <<<<<< start test_multi_task_reduce_and_restart_worker >>>>>>
[Sun Apr 28 15:38:13 CST 2024] <<<<<< start test_multi_task_running >>>>>>
3 dm-master alive
5 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator from time which is in range of mysql binlog(gtid=true)
wait process dm-master.test exit...
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
check log contain failed 1-th time, retry later
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
clean source table
wait process dm-master.test exit...
process dm-master.test already exit
check log contain failed 2-th time, retry later
wait process dm-worker.test exit...
wait process dm-worker.test exit...
check log contain failed 1-th time, retry later
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:38:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
import prepare data
start DM worker and master
[Sun Apr 28 15:38:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>>
[Sun Apr 28 15:38:27 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>>
[Sun Apr 28 15:38:27 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:38:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
check log contain failed 1-th time, retry later
rpc addr 127.0.0.1:8361 is alive
rpc addr 127.0.0.1:8461 is alive
operate mysql config to worker
[Sun Apr 28 15:38:29 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:38:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
dmctl test cmd: "query-status test"
got=1 expected=1
<<<<<< test_source_and_target_with_empty_gtid success! >>>>>>
1 dm-master alive
1 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:38:30 CST 2024] <<<<<< test case all_mode success! >>>>>>
[Pipeline] }
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.1-pull_dm_integration_test-193/tiflow-dm already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
restart worker1
restart dm-worker1
[Pipeline] }
dmctl test cmd: "config source mysql-replica-01"
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
got=1 expected=1
got=1 expected=1
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Sun Apr 28 15:38:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml"
wait process worker1 exit...
process worker1 already exit
[Sun Apr 28 15:38:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
rpc addr 127.0.0.1:8262 is alive
check log contain failed 1-th time, retry later
[Sun Apr 28 15:38:34 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>>
wait for rpc addr 127.0.0.1:8264 alive the 1-th time
rpc addr 127.0.0.1:8264 is alive
[Sun Apr 28 15:38:35 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>>
wait for rpc addr 127.0.0.1:18262 alive the 1-th time
check log contain failed 1-th time, retry later
rpc addr 127.0.0.1:18262 is alive
[Sun Apr 28 15:38:36 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>>
wait for rpc addr 127.0.0.1:18263 alive the 1-th time
dmctl test cmd: "validation start --start-time '2024-04-28 09:38:36' test"
rpc addr 127.0.0.1:18263 is alive
start DM task
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml "
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml "
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
check log contain failed 1-th time, retry later
dmctl test cmd: "query-status test"
got=2 expected=2
got=2 expected=2
dmctl test cmd: "query-status test2"
got=2 expected=2
got=2 expected=2
use sync_diff_inspector to check full dump loader
check diff successfully
check diff successfully
flush logs to force rotate binlog file
apply increment data before restart dm-worker to ensure entering increment phase
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
restart worker1
restart dm-worker1
wait process worker1 exit...
process worker1 already exit
[Sun Apr 28 15:38:42 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
got=1 expected=1
got=1 expected=1
got=1 expected=1
check diff successfully
--> start validator from time > max mysql binlog pos
rpc addr 127.0.0.1:8262 is alive
check log contain failed 1-th time, retry later
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
check log contain failed 1-th time, retry later
wait process dm-worker.test exit...
wait process dm-worker.test exit...
use sync_diff_inspector to check increment data
check diff successfully
check diff successfully
[Sun Apr 28 15:38:45 CST 2024] <<<<<< finish test_multi_task_running >>>>>>
start dumping SQLs into source
"worker1", "worker2",
find workers: worker1 for task: test
find workers: worker2 for task: test
find all workers: worker1 worker2 (total: 2)
restart unuse worker3
try to kill worker port 8264
wait process dm-worker3 exit...
process dm-worker3 already exit
dmctl test cmd: "list-member --worker --name=worker3"
got=1 expected=1
start dm-worker3
[Sun Apr 28 15:38:47 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>>
wait for rpc addr 127.0.0.1:8264 alive the 1-th time
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:38:47 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
check log contain failed 1-th time, retry later
rpc addr 127.0.0.1:8264 is alive
restart unuse worker4
try to kill worker port 18262
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
restart master
restart dm-master
wait process dm-worker4 exit...
process dm-worker4 already exit
dmctl test cmd: "list-member --worker --name=worker4"
got=1 expected=1
start dm-worker4
[Sun Apr 28 15:38:49 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>>
wait for rpc addr 127.0.0.1:18262 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:38:49 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-master exit...
process dm-master already exit
rpc addr 127.0.0.1:18262 is alive
restart unuse worker5
try to kill worker port 18263
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:38:51 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-worker5 exit...
process dm-worker5 already exit
dmctl test cmd: "list-member --worker --name=worker5"
got=1 expected=1
start dm-worker5
[Sun Apr 28 15:38:52 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>>
wait for rpc addr 127.0.0.1:18263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
[Sun Apr 28 15:38:52 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:18263 is alive
try to kill worker port 8262
dmctl test cmd: "config source mysql-replica-01"
got=1 expected=1
got=1 expected=1
wait process dm-worker1 exit...
process dm-worker1 already exit
worker1 was killed
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test2"
got=2 expected=2
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
dmctl test cmd: "validation start --start-time '2026-01-01 00:00:00' test"
rpc addr 127.0.0.1:8261 is alive
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> single varchar col pk
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
restart worker1
restart dm-worker1
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process worker1 exit...
wait process worker1 exit...
wait process dm-worker.test exit...
wait process worker1 exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:38:59 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process worker1 exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:39:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process worker1 exit...
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:39:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process worker1 exit...
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
wait process worker1 exit...
wait process worker1 exit...
wait process worker1 exit...
process worker1 already exit
[Sun Apr 28 15:39:05 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/shardddl4_1/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
rpc addr 127.0.0.1:8262 is alive
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> single datetime col pk
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:14 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:39:15 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:39:17 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 15:39:24 CST 2024] <<<<<< finish DM-155 optimistic >>>>>>
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> compound pk (datetime, timestamp, int, varchar)
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:29 CST 2024] <<<<<< test case shardddl4_1 success! >>>>>>
start running case: [sharding] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sharding/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sharding/run.sh...
Verbose mode = false
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:29 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:39:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:39:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/sharding/source1.yaml"
[Sun Apr 28 15:39:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:39:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/sharding/source2.yaml"
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sharding/conf/dm-task.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=2 expected=2
check sync diff for full dump and load
check diff successfully
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "resume-task test"
check sync diff for the first increment replication
check diff failed 1-th time, retry later
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 4\/4\/3" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
check diff successfully
check sync diff for the second increment replication
check diff successfully
check sync diff for the third increment replication
check diff successfully
checksum before drop/truncate: checksum: 2273109362, checksum after drop/truncate: checksum: 2273109362
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "stop-task test"
dmctl test cmd: "stop-task test"
got=1 expected=1
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> table without primary key
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:44 CST 2024] <<<<<< test case sharding success! >>>>>>
start running case: [sequence_sharding] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/run.sh...
Verbose mode = false
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:39:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:39:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 15:39:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:39:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding/source1.yaml"
dmctl test cmd: "operate-source create /tmp/dm_test/sequence_sharding/source2.yaml"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/sequence_sharding/conf/dm-task.yaml "
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
check diff successfully
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=0 expected=1
command: validation status test "stage": "Stopped" count: 0 != expected: 1, failed the 0-th time, will retry again
check diff successfully
dmctl test cmd: "query-status sequence_sharding"
got=0 expected=2
command: query-status sequence_sharding detect inconsistent DDL sequence count: 0 != expected: 2, failed the 0-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> table is deleted on downstream
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
got=2 expected=2
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:39:58 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:39:59 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:40:00 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test"
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> table in schema-tracker has less column than binlog
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:40:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
dmctl test cmd: "resume-task sequence_sharding"
dmctl test cmd: "query-status sequence_sharding"
got=2 expected=2
dmctl test cmd: "stop-task sequence_sharding"
dmctl test cmd: "start-task /tmp/dm_test/sequence_sharding/task.yaml"
dmctl test cmd: "query-status sequence_sharding"
got=2 expected=2
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-master.test exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:40:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
process dm-master.test already exit
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
wait process dm-worker.test exit...
[Sun Apr 28 15:40:11 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-worker.test exit...
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:40:12 CST 2024] <<<<<< test case sequence_sharding success! >>>>>>
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.1-pull_dm_integration_test-193/tiflow-dm already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test"
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> pk column of downstream table not in range of binlog column
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:40:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:40:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:40:21 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone-no-validator.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "validation start --start-time '2022-01-01 00:00:00' test"
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
--> stopped validator fail over
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:40:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
use sync_diff_inspector to check increment data
check diff successfully
check diff successfully
data checked after one worker was killed
try to kill worker port 8263
wait process dm-worker2 exit...
process dm-worker2 already exit
worker2 was killed
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test2"
got=2 expected=2
[Sun Apr 28 15:40:28 CST 2024] <<<<<< finish test_multi_task_reduce_and_restart_worker >>>>>>
3 dm-master alive
3 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
wait process dm-master.test exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:40:31 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-master.test exit...
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:40:32 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-master.test exit...
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:40:36 CST 2024] <<<<<< test case ha_cases2 success! >>>>>>
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/2\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.1-pull_dm_integration_test-193/tiflow-dm already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=2 expected=2
run tidb sql failed 1-th time, retry later
dmctl test cmd: "validation stop test"
got=1 expected=1
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
wait process worker1 exit...
process worker1 already exit
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=0 expected=0
got=1 expected=1
dmctl test cmd: "validation show-error --error all test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "validation start test"
got=1 expected=1
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 6\/3\/1" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=2 expected=2
--> filter online ddl shadow table
1 dm-master alive
1 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:40:57 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:40:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:40:59 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/test-filter.yaml --remove-meta"
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=2 expected=2
--> filter by ba list
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=2 expected=2
--> filter by filter-rules
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test "processedRowsStatus": "insert\/update\/delete: 7\/2\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=0 expected=1
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 1, failed the 1-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=3 expected=3
--> validate when syncer is stopped
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:41:19 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:41:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:41:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
ERROR: Failed to launch dm-it-39db2a2f-981e-424d-a78a-2133a6e6bc2e-jb53w-3d18k
io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[dm-it-39db2a2f-981e-424d-a78a-2133a6e6bc2e-jb53w-3d18k] in namespace [jenkins-tiflow].
	at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939)
	at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921)
	at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97)
	at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185)
	at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297)
	at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46)
	at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at java.base/java.lang.Thread.run(Thread.java:829)
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-task-standalone.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=1 expected=1
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again
got=1 expected=1
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again
got=1 expected=1
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again
got=1 expected=1
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again
got=1 expected=1
got=1 expected=1
dmctl test cmd: "pause-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "pause-task test"
dmctl test cmd: "query-status test"
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 2\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again
got=1 expected=1
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "pause-task test"
dmctl test cmd: "query-status test"
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again
got=1 expected=1
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "pause-task test"
dmctl test cmd: "query-status test"
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 4\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again
got=1 expected=1
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "pause-task test"
dmctl test cmd: "query-status test"
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 5\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again
got=1 expected=1
got=1 expected=1
dmctl test cmd: "resume-task test"
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "query-status test"
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 1-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 2-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 3-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 4-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 5-th time, will retry again
got=0 expected=1
command: query-status test "processedRowsStatus": "insert\/update\/delete: 6\/0\/0" count: 0 != expected: 1, failed the 6-th time, will retry again
got=1 expected=1
got=1 expected=1
--> test duplicate auto-incr pk
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:43:18 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:43:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source1.yaml"
[Sun Apr 28 15:43:22 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/source2.yaml"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_basic/conf/sharding-task.yaml --remove-meta"
dmctl test cmd: "query-status test"
got=1 expected=2
command: query-status test "processedRowsStatus": "insert\/update\/delete: 3\/0\/0" count: 1 != expected: 2, failed the 0-th time, will retry again
got=2 expected=2
got=2 expected=2
got=2 expected=2
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:43:35 CST 2024] <<<<<< test case validator_basic success! >>>>>>
restore config
restore time_zone
start running case: [validator_checkpoint] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/run.sh...
Verbose mode = false
--> check persist checkpoint and data with 2 source
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:43:35 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:43:37 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 15:43:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml"
dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta"
dmctl test cmd: ""unit": "Sync""
dmctl test cmd: "validation status test"
got=0 expected=1
command: validation status test processedRowsStatus": "insert\/update\/delete: 3\/1\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=2 expected=2
--> check validator can restart from previous position on fail over
restart dm-worker1
wait process worker1 exit...
process worker1 already exit
[Sun Apr 28 15:43:45 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
restart dm-worker2
wait process worker2 exit...
process worker2 already exit
[Sun Apr 28 15:43:47 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "validation status test"
got=1 expected=1
got=0 expected=1
command: validation status test processedRowsStatus": "insert\/update\/delete: 1\/0\/0" count: 0 != expected: 1, failed the 0-th time, will retry again
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=1 expected=1
got=2 expected=2
dmctl test cmd: "validation status test"
got=1 expected=1
got=1 expected=1
got=1 expected=2
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 1 != expected: 2, failed the 0-th time, will retry again
got=1 expected=1
got=1 expected=1
got=2 expected=2
got=2 expected=2
run tidb sql failed 1-th time, retry later
--> check validator persist fail in the middle
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:44:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 15:44:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 15:44:02 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source1.yaml"
dmctl test cmd: "operate-source create /tmp/dm_test/validator_checkpoint/source2.yaml"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml --remove-meta"
dmctl test cmd: ""unit": "Sync""
dmctl test cmd: "validation status test"
got=0 expected=2
command: validation status test "stage": "Stopped" count: 0 != expected: 2, failed the 0-th time, will retry again
got=2 expected=2
got=2 expected=2
got=2 expected=2
got=2 expected=2
got=2 expected=2
--> after resume and persist again, meta data should be consistent
dmctl test cmd: "stop-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.1/pull_dm_integration_test/tiflow/dm/tests/validator_checkpoint/conf/dm-task.yaml "
dmctl test cmd: "validation status test"
got=4 expected=4
got=2 expected=2
got=2 expected=2
got=2 expected=2
run tidb sql failed 1-th time, retry later
--> check validate success after insert data manually
dmctl test cmd: "validation status test"
got=4 expected=4
got=2 expected=2
got=0 expected=2
command: validation status test pendingRowsStatus": "insert\/update\/delete: 0\/0\/0 count: 0 != expected: 2, failed the 0-th time, will retry again
got=4 expected=4
got=2 expected=2
got=2 expected=2
got=2 expected=2
run tidb sql failed 1-th time, retry later
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 15:44:21 CST 2024] <<<<<< test case validator_checkpoint success! >>>>>>
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.1-pull_dm_integration_test-193/tiflow-dm already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // parallel
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
Finished: SUCCESS