Skip to content
[Jenkins]Jenkins
log in

Console Output

Skipping 646 KB.. Full Log
alter: add column c int comment '1  2
3😊4';
1.1.0
dmctl test cmd: "query-status test"
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:18:28 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>>
wait for rpc addr 127.0.0.1:8361 alive the 1-th time
got=2 expected=2
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
check diff successfully
check diff failed 1-th time, retry later
check diff failed 1-th time, retry later
dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-04-28 14:18:26'"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml"
rpc addr 127.0.0.1:8361 is alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task-safe-mode-duration.yaml --remove-meta"
dmctl test cmd: "stop-task test"
dmctl test cmd: "stop-relay -s mysql-replica-01"
dmctl test cmd: "query-status -s mysql-replica-01"
alter: add column c int comment '1  2
3😊4';
pt-online-schema-change 3.0.12
[Sun Apr 28 20:18:30 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
current leader is master4
dmctl test cmd: "list-member --master"
got=5 expected=5
[Sun Apr 28 20:18:31 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS pessimistic >>>>>>
[Sun Apr 28 20:18:31 CST 2024] <<<<<< start DM-ADD_DROP_COLUMNS optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta"
current leader is master4
kill master1
check diff successfully
check diff failed 2-th time, retry later
*******************************************************************
 Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client
 is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER
 possibly with SSL_ca_file|SSL_ca_path for verification.
 If you really don't want to verify the certificate and keep the
 connection open to Man-In-The-Middle attacks please set
 SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application.
*******************************************************************
  at /usr/bin/pt-online-schema-change line 6976.
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "transfer-source mysql-replica-01 worker2"
dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2024-04-28 14:18:26'"
wait process dm-master1 exit...
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
dmctl test cmd: "stop-task test"
wait process dm-master1 exit...
process dm-master1 already exit
check diff failed 3-th time, retry later
*******************************************************************
 Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client
 is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER
 possibly with SSL_ca_file|SSL_ca_path for verification.
 If you really don't want to verify the certificate and keep the
 connection open to Man-In-The-Middle attacks please set
 SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application.
*******************************************************************
  at /usr/bin/pt-online-schema-change line 6976.
wait process dm-worker.test exit...
process dm-worker.test already exit
[Sun Apr 28 20:18:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>>
[Sun Apr 28 20:18:33 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '1995-03-07 01:02:03'"
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "query-status test"
alter: add column c int comment '1  2
3😊4';
pt-online-schema-change 3.0.12
got=1 expected=1
dmctl test cmd: "stop-task test"
got=3 expected=3
dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml "
check diff successfully
check diff failed 1-th time, retry later
alter: add column c int comment '1  2
3😊4';
pt-online-schema-change 3.0.12
check diff failed at last
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:18:35 CST 2024] <<<<<< finish DM-111 optimistic >>>>>>
dmctl test cmd: "stop-task test"
dmctl test cmd: "start-task /tmp/dm_test/start_task/dm-task.yaml --start-time '2037-12-12 01:02:03'"
*******************************************************************
 Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client
 is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER
 possibly with SSL_ca_file|SSL_ca_path for verification.
 If you really don't want to verify the certificate and keep the
 connection open to Man-In-The-Middle attacks please set
 SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application.
*******************************************************************
  at /usr/bin/pt-online-schema-change line 6976.
check diff successfully
dmctl test cmd: "stop-task test"
dmctl test cmd: "query-status test"
got=3 expected=3
dmctl test cmd: "start-task /tmp/dm_test/safe_mode/dm-task-safe-mode-duration.yaml "
got=1 expected=1
got=1 expected=1
dmctl test cmd: "stop-task test"
1 dm-master alive
[Sun Apr 28 20:18:36 CST 2024] <<<<<< start DM-112 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta"
2 dm-worker alive
0 dm-syncer alive
check diff successfully
*******************************************************************
 Using the default of SSL_verify_mode of SSL_VERIFY_NONE for client
 is deprecated! Please set SSL_verify_mode to SSL_VERIFY_PEER
 possibly with SSL_ca_file|SSL_ca_path for verification.
 If you really don't want to verify the certificate and keep the
 connection open to Man-In-The-Middle attacks please set
 SSL_verify_mode explicitly to SSL_VERIFY_NONE in your application.
*******************************************************************
  at /usr/bin/pt-online-schema-change line 6976.
check diff successfully
check diff successfully
dmctl test cmd: "stop-task test"
wait process dm-master.test exit...
[Sun Apr 28 20:18:37 CST 2024] <<<<<< finish DM-ADD_DROP_COLUMNS optimistic >>>>>>
[Sun Apr 28 20:18:37 CST 2024] <<<<<< start DM-COLUMN_INDEX pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-pessimistic.yaml --remove-meta"
check diff successfully
finish running safe mode duration case
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
check diff failed 1-th time, retry later
wait for dm to sync
use sync_diff_inspector to check data now!
check diff successfully
[Sun Apr 28 20:18:37 CST 2024] <<<<<< finish test_kill_worker_in_sync >>>>>>
3 dm-master alive
2 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
[Sun Apr 28 20:18:38 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master1.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
alter: add column c int comment '1  2
3😊4';
pt-online-schema-change 3.0.12
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-master.test exit...
[Sun Apr 28 20:18:38 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
rpc addr 127.0.0.1:8261 is alive
dmctl test cmd: "list-member --master"
got=5 expected=5
check diff failed 1-th time, retry later
wait process dm-master.test exit...
current leader is master4
kill master2
wait process dm-worker.test exit...
rpc addr 127.0.0.1:8263 is alive
use sync_diff_inspector to check full data
check diff successfully
alter: add column age int;
1.1.0
wait process dm-master.test exit...
process dm-master.test already exit
check diff failed 2-th time, retry later
wait process dm-master.test exit...
wait process dm-master2 exit...
wait process dm-worker.test exit...
wait process dm-worker.test exit...
wait process dm-master.test exit...
wait process dm-master2 exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
alter: add column age int;
1.1.0
process dm-master2 already exit
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:18:42 CST 2024] <<<<<< finish DM-COLUMN_INDEX pessimistic >>>>>>
[Sun Apr 28 20:18:42 CST 2024] <<<<<< start DM-COLUMN_INDEX optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:18:42 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-master.test exit...
check diff failed 3-th time, retry later
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
wait process dm-master.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
[Sun Apr 28 20:18:43 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:18:44 CST 2024] <<<<<< finish DM-COLUMN_INDEX optimistic >>>>>>
check diff failed at last
dmctl test cmd: "stop-task test"
alter: add key name (name);
1.1.0
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:18:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
[Sun Apr 28 20:18:44 CST 2024] <<<<<< finish DM-112 optimistic >>>>>>
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:18:44 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-master.test exit...
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/start_task/source1.yaml"
[Sun Apr 28 20:18:45 CST 2024] <<<<<< start DM-113 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 20:18:45 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-master.test exit...
wait process dm-worker exit...
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml --remove-meta"
alter: add key name (name);
1.1.0
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source1.yaml"
wait process dm-master.test exit...
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:18:46 CST 2024] <<<<<< test case ha_cases_2 success! >>>>>>
start running case: [ha_cases2] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/run.sh...
Verbose mode = false
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:18:46 CST 2024] <<<<<< start test_pause_task >>>>>>
[Sun Apr 28 20:18:46 CST 2024] <<<<<< start test_multi_task_running >>>>>>
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
clean source table
wait process dm-worker exit...
process dm-worker already exit
[Sun Apr 28 20:18:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>>
[Sun Apr 28 20:18:46 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
check diff failed 1-th time, retry later
[Sun Apr 28 20:18:46 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master2.toml >>>>>>
wait for rpc addr 127.0.0.1:8361 alive the 1-th time
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8263 is alive
[Sun Apr 28 20:18:47 CST 2024] <<<<<< start DM-DML_EXECUTE_ERROR  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta"
dmctl test cmd: "operate-source create /tmp/dm_test/safe_mode/source2.yaml"
rpc addr 127.0.0.1:8361 is alive
dmctl test cmd: "list-member --master"
got=5 expected=5
current leader is master4
kill master3
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-task.yaml --remove-meta"
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
1.1.0
dmctl test cmd: "query-status test"
got=1 expected=1
check log contain failed 1-th time, retry later
import prepare data
start DM worker and master
[Sun Apr 28 20:18:48 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master1.toml >>>>>>
[Sun Apr 28 20:18:49 CST 2024] <<<<<< START DM-MASTER on port 8361, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master2.toml >>>>>>
[Sun Apr 28 20:18:49 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-master3.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:18:49 CST 2024] <<<<<< finish DM-113 pessimistic >>>>>>
[Sun Apr 28 20:18:49 CST 2024] <<<<<< start DM-113 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-master3 exit...
rpc addr 127.0.0.1:8261 is alive
check diff successfully
wait for port 8262 offline the 1-th time
rpc addr 127.0.0.1:8361 is alive
rpc addr 127.0.0.1:8461 is alive
operate mysql config to worker
[Sun Apr 28 20:18:50 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
check diff successfully
wait process dm-master3 exit...
process dm-master3 already exit
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
1.1.0
dmctl test cmd: "query-status test"
check diff successfully
dmctl test cmd: "stop-task test"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml "
dmctl test cmd: "query-status test"
got=2 expected=2
got=1 expected=1
got=1 expected=1
dmctl test cmd: "stop-task test"
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source1.yaml"
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:18:51 CST 2024] <<<<<< finish DM-DML_EXECUTE_ERROR  >>>>>>
[Sun Apr 28 20:18:51 CST 2024] <<<<<< finish DM-113 optimistic >>>>>>
wait for port 8262 offline the 2-th time
[Sun Apr 28 20:18:52 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>>
[Sun Apr 28 20:18:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
[Sun Apr 28 20:18:52 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-worker exit...
[Sun Apr 28 20:18:52 CST 2024] <<<<<< start DM-114 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta"
alter: add column age int;
1.1.0
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8263 is alive
wait process dm-worker exit...
process dm-worker already exit
[Sun Apr 28 20:18:53 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>>
[Sun Apr 28 20:18:53 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/ha_cases2/source2.yaml"
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8263 is alive
[Sun Apr 28 20:18:54 CST 2024] <<<<<< start DM-KEY_NOT_FOUND  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/double-source-optimistic.yaml --remove-meta"
[Sun Apr 28 20:18:55 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker3.toml >>>>>>
wait for rpc addr 127.0.0.1:8264 alive the 1-th time
check diff successfully
1 dm-master alive
1 dm-worker alive
0 dm-syncer alive
alter: add column age int;
1.1.0
wait process dm-master.test exit...
dmctl test cmd: "query-status test"
[Sun Apr 28 20:18:55 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>>
got=2 expected=2
check diff successfully
wait for rpc addr 127.0.0.1:8461 alive the 1-th time
rpc addr 127.0.0.1:8264 is alive
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:18:56 CST 2024] <<<<<< START DM-WORKER on port 18262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker4.toml >>>>>>
wait for rpc addr 127.0.0.1:18262 alive the 1-th time
wait process dm-master.test exit...
process dm-master.test already exit
[Sun Apr 28 20:18:56 CST 2024] <<<<<< finish DM-KEY_NOT_FOUND  >>>>>>
rpc addr 127.0.0.1:8461 is alive
dmctl test cmd: "list-member --master"
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:18:56 CST 2024] <<<<<< finish DM-114 pessimistic >>>>>>
[Sun Apr 28 20:18:56 CST 2024] <<<<<< start DM-114 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta"
got=5 expected=5
current leader is master4
rpc addr 127.0.0.1:18262 is alive
[Sun Apr 28 20:18:57 CST 2024] <<<<<< START DM-WORKER on port 18263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-worker5.toml >>>>>>
wait for rpc addr 127.0.0.1:18263 alive the 1-th time
current leader is master4
kill master5
wait process dm-worker exit...
alter: add key name (name);
1.1.0
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
rpc addr 127.0.0.1:18263 is alive
start DM task
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task.yaml "
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_cases2/conf/dm-task2.yaml "
wait process dm-master5 exit...
check diff failed 1-th time, retry later
wait process dm-worker exit...
process dm-worker already exit
[Sun Apr 28 20:18:58 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker1.toml >>>>>>
[Sun Apr 28 20:18:58 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-worker.test exit...
check sync diff after set SafeModeInitPhaseSeconds failpoint
check diff successfully
wait for one of DM-worker offine failed, retry later
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
wait process dm-master5 exit...
process dm-master5 already exit
alter: add key name (name);
1.1.0
failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchTargetDoTablesFailed=return(1152)
wait for one of DM-worker offine failed, retry later
dmctl test cmd: "query-status test"
got=2 expected=2
got=2 expected=2
dmctl test cmd: "query-status test2"
got=2 expected=2
got=2 expected=2
use sync_diff_inspector to check full dump loader
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8263 is alive
[Sun Apr 28 20:18:59 CST 2024] <<<<<< start DM-001  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-no-sharding.yaml --remove-meta"
check diff successfully
check diff successfully
flush logs to force rotate binlog file
apply increment data before restart dm-worker to ensure entering increment phase
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:00 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
DM-worker1 is sharding lock owner and detects it offline
[Sun Apr 28 20:19:00 CST 2024] <<<<<< finish DM-114 optimistic >>>>>>
[Sun Apr 28 20:19:00 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/safe_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "query-status test"
got=1 expected=1
dmctl test cmd: "query-status test"
got=2 expected=2
got=1 expected=1
dmctl test cmd: "stop-task test"
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:01 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
[Sun Apr 28 20:19:01 CST 2024] <<<<<< finish DM-001  >>>>>>
[Sun Apr 28 20:19:01 CST 2024] <<<<<< start DM-115 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8262 is alive
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
1.1.0
[Sun Apr 28 20:19:02 CST 2024] <<<<<< start DM-002  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/start_task/0/source1.yaml"
check un-accessible DM-worker exists
dmctl test cmd: "query-status -s 127.0.0.1:8888"
start task and will failed
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml"
reset go failpoints, and need restart dm-worker, then start task again
dmctl test cmd: "query-status test"
got=2 expected=2
check log contain failed 1-th time, retry later
dmctl test cmd: "query-status test"
got=1 expected=1
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
1.1.0
wait process dm-worker.test exit...
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:04 CST 2024] <<<<<< finish DM-002  >>>>>>
[Sun Apr 28 20:19:04 CST 2024] <<<<<< START DM-MASTER on port 8661, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master5.toml >>>>>>
wait for rpc addr 127.0.0.1:8661 alive the 1-th time
[Sun Apr 28 20:19:05 CST 2024] <<<<<< start DM-003 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta"
wait process dm-worker.test exit...
process dm-worker.test already exit
rpc addr 127.0.0.1:8661 is alive
dmctl test cmd: "list-member --master"
check diff successfully
dmctl test cmd: "stop-task test"
got=5 expected=5
dmctl test cmd: "list-member --worker --name=worker1,worker2"
[Sun Apr 28 20:19:06 CST 2024] <<<<<< finish DM-115 optimistic >>>>>>
got=2 expected=2
dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source1.yaml"
dmctl test cmd: "list-member --worker"
wait process dm-master.test exit...
alter: add column age int;
pt-online-schema-change 3.0.12
got=1 expected=1
got=1 expected=1
dmctl test cmd: "operate-source stop /tmp/dm_test/ha_master/source2.yaml"
use sync_diff_inspector to check increment data
check diff successfully
check diff successfully
[Sun Apr 28 20:19:05 CST 2024] <<<<<< finish test_multi_task_running >>>>>>
dmctl test cmd: "start-relay -s mysql-replica-01 worker1"
dmctl test cmd: "query-status test"
got=1 expected=1
alter: add column age int;
dmctl test cmd: "list-member"
got=2 expected=2
dmctl test cmd: "start-relay -s mysql-replica-02 worker2"
[Sun Apr 28 20:19:07 CST 2024] <<<<<< start DM-116 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta"
check diff failed 1-th time, retry later
pt-online-schema-change 3.0.12
got=2 expected=2
dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source1.yaml"
alter: add key name (name);
wait process dm-master.test exit...
process dm-master.test already exit
[Sun Apr 28 20:19:07 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
check sync diff after restart DDL owner
check diff successfully
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
pt-online-schema-change 3.0.12
alter: add key name (name);
wait process dm-master.test exit...
pt-online-schema-change 3.0.12
got=2 expected=2
start dumping SQLs into source
pause tasks test
dmctl test cmd: "pause-task test"
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "operate-source create /tmp/dm_test/ha_master/source2.yaml"
dmctl test cmd: "pause-task test"
pt-online-schema-change 3.0.12
check diff failed 1-th time, retry later
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
dmctl test cmd: "query-status test"
check diff failed 2-th time, retry later
pause tasks test2
dmctl test cmd: "pause-task test2"
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
pt-online-schema-change 3.0.12
wait process dm-master.test exit...
process dm-master.test already exit
dmctl test cmd: "pause-task test2"
dmctl test cmd: "list-member --name=worker1,worker2"
dmctl test cmd: "query-status test2"
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:09 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
alter: add column age int;
pt-online-schema-change 3.0.12
got=2 expected=2
kill worker1
wait process dm-worker.test exit...
alter: add column age int;
pt-online-schema-change 3.0.12
rpc addr 127.0.0.1:8262 is alive
alter: add key name (name);
wait process dm-worker1 exit...
pt-online-schema-change 3.0.12
resume tasks test
dmctl test cmd: "resume-task test"
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:11 CST 2024] <<<<<< finish DM-116 pessimistic >>>>>>
[Sun Apr 28 20:19:11 CST 2024] <<<<<< start DM-116 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-worker.test exit...
got=3 expected=3
dmctl test cmd: "resume-task test"
alter: add key name (name);
pt-online-schema-change 3.0.12
check diff failed 3-th time, retry later
got=3 expected=3
dmctl test cmd: "query-status test"
resume tasks test2
dmctl test cmd: "resume-task test2"
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
pt-online-schema-change 3.0.12
got=3 expected=3
dmctl test cmd: "resume-task test2"
wait process dm-worker1 exit...
process dm-worker1 already exit
got=3 expected=3
dmctl test cmd: "query-status test2"
wait process dm-worker.test exit...
dmctl test cmd: "list-member --name=worker1,worker2"
got=1 expected=1
got=1 expected=1
kill worker2
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:12 CST 2024] <<<<<< test case safe_mode success! >>>>>>
start running case: [sequence_safe_mode] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/run.sh...
Verbose mode = false
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
alter: add column info_json json GENERATED ALWAYS AS (`info`) VIRTUAL;
dmctl test cmd: "query-status test"
got=2 expected=2
[Sun Apr 28 20:19:12 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
pt-online-schema-change 3.0.12
check diff successfully
dmctl test cmd: "stop-task test"
wait process dm-worker2 exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:13 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
use sync_diff_inspector to check increment data
check diff successfully
start dm-worker3 and kill dm-worker2
[Sun Apr 28 20:19:13 CST 2024] <<<<<< finish DM-116 optimistic >>>>>>
check diff failed at last
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:13 CST 2024] <<<<<< finish DM-003 pessimistic >>>>>>
wait process dm-worker2 exit...
[Sun Apr 28 20:19:14 CST 2024] <<<<<< start DM-117 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-worker2 exit...
process dm-worker2 already exit
dmctl test cmd: "list-member"
got=2 expected=2
[Sun Apr 28 20:19:14 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
[Sun Apr 28 20:19:14 CST 2024] <<<<<< start DM-004 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-optimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 20:19:14 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-worker2 exit...
rpc addr 127.0.0.1:8262 is alive
[Sun Apr 28 20:19:15 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-worker2.toml >>>>>>
process dm-worker2 already exit
[Sun Apr 28 20:19:15 CST 2024] <<<<<< START DM-WORKER on port 8264, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/online_ddl/conf/dm-worker3.toml >>>>>>
wait for rpc addr 127.0.0.1:8264 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source1.yaml"
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "operate-source create /tmp/dm_test/sequence_safe_mode/source2.yaml"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml "
dmctl test cmd: "query-status test"
got=1 expected=1
got=1 expected=1
dmctl test cmd: "binlog-schema update test shardddl1 tb1 /tmp/dm_test/shardddl3_1/schema.sql -s mysql-replica-01"
dmctl test cmd: "query-status test"
got=1 expected=1
run tidb sql failed 1-th time, retry later
dmctl test cmd: "binlog skip test"
got=2 expected=2
got=1 expected=1
rpc addr 127.0.0.1:8264 is alive
wait and check task running
dmctl test cmd: "query-status test"
check diff successfully
dmctl test cmd: "stop-task test"
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "list-member --worker"
got=3 expected=3
[Sun Apr 28 20:19:16 CST 2024] <<<<<< finish DM-117 optimistic >>>>>>
got=2 expected=2
dmctl test cmd: "config master master1"
got=1 expected=1
dmctl test cmd: "config master master2"
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-task.yaml --remove-meta"
check diff successfully
1 dm-master alive
1 dm-worker alive
0 dm-syncer alive
got=1 expected=1
[Sun Apr 28 20:19:17 CST 2024] <<<<<< finish test_list_member_command >>>>>>
dmctl test cmd: "start-relay -s mysql-replica-01 worker1"
[Sun Apr 28 20:19:17 CST 2024] <<<<<< start DM-118 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-master.test exit...
check diff successfully
wait for port 8262 offline the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:18 CST 2024] <<<<<< finish DM-004 optimistic >>>>>>
got=2 expected=2
start DM task
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-task.yaml --remove-meta"
wait process dm-master.test exit...
process dm-master.test already exit
use sync_diff_inspector to check increment2 data now!
check diff successfully
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
wait for port 8262 offline the 2-th time
[Sun Apr 28 20:19:19 CST 2024] <<<<<< start DM-005  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1/conf/single-source-pessimistic.yaml --remove-meta"
wait process dm-worker.test exit...
use sync_diff_inspector to check full dump loader
check diff successfully
flush logs to force rotate binlog file
kill dm-master1 and kill dm-master2
wait process dm-master.test exit...
[Sun Apr 28 20:19:20 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>>
[Sun Apr 28 20:19:20 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "query-status test"
got=1 expected=1
wait process dm-worker.test exit...
wait process dm-master.test exit...
process dm-master.test already exit
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:21 CST 2024] <<<<<< finish DM-005  >>>>>>
wait process dm-master1 exit...
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8263 is alive
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:21 CST 2024] <<<<<< finish DM-118 pessimistic >>>>>>
[Sun Apr 28 20:19:21 CST 2024] <<<<<< start DM-118 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl3_1/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-master1 exit...
process dm-master1 already exit
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
failpoint=github.com/pingcap/tiflow/dm/pkg/conn/FetchAllDoTablesFailed=return(1152)
wait process dm-worker.test exit...
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 20:19:22 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-master2 exit...
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
wait process dm-master.test exit...
check sync diff after reset failpoint
check diff successfully
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:23 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:24 CST 2024] <<<<<< test case online_ddl success! >>>>>>
wait process dm-master2 exit...
process dm-master2 already exit
wait and check task running
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-293/tiflow-dm already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
HTTP 127.0.0.1:8461/apis/v1alpha1/status/test is alive
dmctl test cmd: "query-status test"
got=3 expected=3
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
wait process dm-worker.test exit...
wait process dm-master.test exit...
process dm-master.test already exit
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml"
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
wait process dm-worker.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
restart DM-worker after set SequenceShardSyncedExecutionExit failpoint
[Sun Apr 28 20:19:25 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>>
[Sun Apr 28 20:19:25 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:25 CST 2024] <<<<<< finish DM-118 optimistic >>>>>>
check un-accessible DM-worker exists
dmctl test cmd: "query-status -s 127.0.0.1:8888"
start task and will failed
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml"
reset go failpoints, and need restart dm-worker, then start task again
wait process dm-worker.test exit...
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8263 is alive
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
use sync_diff_inspector to check data now!
check diff successfully
dmctl test cmd: "offline-member --master --name master1"
dmctl test cmd: "offline-member --master --name master2"
kill dm-master3
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:27 CST 2024] <<<<<< test case shardddl1 success! >>>>>>
start running case: [shardddl1_1] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/run.sh...
Verbose mode = false
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:27 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-worker.test exit...
wait process dm-master3 exit...
wait process dm-master.test exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:28 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker1.toml >>>>>>
wait process dm-worker.test exit...
process dm-worker.test already exit
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait process dm-master3 exit...
process dm-master3 already exit
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-master.test exit...
wait for one of DM-worker offine failed, retry later
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source1.yaml"
wait for one of DM-worker offine failed, retry later
[Sun Apr 28 20:19:29 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
wait process dm-worker.test exit...
wait process dm-master.test exit...
process dm-master.test already exit
[Sun Apr 28 20:19:30 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/shardddl1_1/source2.yaml"
wait process dm-worker.test exit...
DM-worker1 is sharding lock owner and detects it offline
[Sun Apr 28 20:19:30 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/sequence_safe_mode/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
wait for rpc addr 127.0.0.1:8261 alive the 2-th time
dmctl test cmd: "query-status test"
got=3 expected=3
[Sun Apr 28 20:19:31 CST 2024] <<<<<< START DM-MASTER on port 8461, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master3.toml >>>>>>
wait for rpc addr 127.0.0.1:8461 alive the 1-th time
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:32 CST 2024] <<<<<< test case shardddl3_1 success! >>>>>>
start running case: [shardddl4] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/run.sh...
Verbose mode = false
rpc addr 127.0.0.1:8262 is alive
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:32 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
[Sun Apr 28 20:19:32 CST 2024] <<<<<< start DM-SAME_DDL_TWICE pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8461 is alive
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:32 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:33 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
rpc addr 127.0.0.1:8262 is alive
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source1.yaml"
[Sun Apr 28 20:19:34 CST 2024] <<<<<< START DM-WORKER on port 8263, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/dm-worker2.toml >>>>>>
wait for rpc addr 127.0.0.1:8263 alive the 1-th time
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8263 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/shardddl4/source2.yaml"
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:36 CST 2024] <<<<<< finish DM-SAME_DDL_TWICE pessimistic >>>>>>
[Sun Apr 28 20:19:36 CST 2024] <<<<<< start DM-006  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta"
[Sun Apr 28 20:19:36 CST 2024] <<<<<< start DM-119 pessimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-pessimistic.yaml --remove-meta"
check sync diff after restart DDL owner
check diff successfully
1 dm-master alive
2 dm-worker alive
0 dm-syncer alive
[Sun Apr 28 20:19:37 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/ha_master/conf/dm-master-join1.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "query-status test"
got=2 expected=2
wait process dm-master.test exit...
check diff failed 1-th time, retry later
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8261 is alive
dmctl test cmd: "pause-task test"
got=3 expected=3
got=1 expected=1
got=1 expected=1
4 dm-master alive
2 dm-worker alive
0 dm-syncer alive
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/start_task/conf/dm-task.yaml "
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-master.test exit...
check diff successfully
1 dm-master alive
1 dm-worker alive
0 dm-syncer alive
wait process dm-worker.test exit...
wait process dm-master.test exit...
check diff failed 2-th time, retry later
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:40 CST 2024] <<<<<< finish DM-006  >>>>>>
wait process dm-master.test exit...
wait process dm-worker.test exit...
wait process dm-master.test exit...
[Sun Apr 28 20:19:41 CST 2024] <<<<<< start DM-007  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-master.test exit...
process dm-master.test already exit
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:42 CST 2024] <<<<<< test case sequence_safe_mode success! >>>>>>
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_dm_integration_test-293/tiflow-dm already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
check diff failed 3-th time, retry later
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
wait process dm-master.test exit...
[Pipeline] // node
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
[Pipeline] }
wait process dm-worker.test exit...
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
wait process dm-master.test exit...
wait process dm-worker.test exit...
check diff failed 4-th time, retry later
wait process dm-master.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:45 CST 2024] <<<<<< start test_COMMIT_in_QueryEvent >>>>>>
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:45 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /tmp/dm_test/start_task/1/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:45 CST 2024] <<<<<< finish DM-007  >>>>>>
wait process dm-master.test exit...
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:19:46 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/start_task/1/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
[Sun Apr 28 20:19:46 CST 2024] <<<<<< start DM-008  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-master.test exit...
check diff failed 5-th time, retry later
rpc addr 127.0.0.1:8262 is alive
dmctl test cmd: "operate-source create /tmp/dm_test/start_task/1/source1.yaml"
got=2 expected=2
got=1 expected=1
check master alive
dmctl test cmd: "list-member"
got=1 expected=1
start task and check stage
dmctl test cmd: "start-task /tmp/dm_test/start_task/1/dm-task.yaml --remove-meta=true"
dmctl test cmd: "query-status test"
got=2 expected=2
wait process dm-master.test exit...
check diff failed 1-th time, retry later
check diff failed 6-th time, retry later
wait process dm-master.test exit...
got=2 expected=2
dmctl test cmd: "query-status test"
got=2 expected=2
got=1 expected=1
got=2 expected=2
check data
check diff failed 1-th time, retry later
wait process dm-master.test exit...
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:50 CST 2024] <<<<<< finish DM-008  >>>>>>
check diff failed 7-th time, retry later
wait process dm-master.test exit...
[Sun Apr 28 20:19:51 CST 2024] <<<<<< start DM-009  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta"
check diff successfully
<<<<<< test_COMMIT_in_QueryEvent success! >>>>>>
1 dm-master alive
1 dm-worker alive
0 dm-syncer alive
wait process dm-master.test exit...
wait process dm-master.test exit...
wait process dm-master.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
wait process dm-master.test exit...
process dm-master.test already exit
check diff failed 8-th time, retry later
wait process dm-master.test exit...
wait process dm-worker.test exit...
wait process dm-master.test exit...
check diff failed 9-th time, retry later
wait process dm-worker.test exit...
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:55 CST 2024] <<<<<< finish DM-009  >>>>>>
wait process dm-master.test exit...
wait process dm-worker.test exit...
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:19:56 CST 2024] <<<<<< test case start_task success! >>>>>>
start running case: [print_status] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh...
Verbose mode = false
+++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/run.sh
++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status
++ pwd
+ cur=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status
+ source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../_utils/test_prepare
++ source env_variables
+++ MYSQL_HOST1=127.0.0.1
+++ MYSQL_HOST2=127.0.0.1
+++ TIDB_HOST=127.0.0.1
+++ MYSQL_PORT1=3306
+++ MYSQL_PORT2=3307
+++ MYSQL_PASSWORD1=123456
+++ MYSQL_PASSWORD2=123456
+++ TIDB_PASSWORD=123456
+++ TIDB_PORT=4000
+++ MASTER_PORT=8261
+++ MASTER_PEER_PORT=8291
+++ MASTER_PORT1=8261
+++ MASTER_PORT2=8361
+++ MASTER_PORT3=8461
+++ MASTER_PORT4=8561
+++ MASTER_PORT5=8661
+++ MASTER_PORT6=8761
+++ MASTER_PEER_PORT1=8291
+++ MASTER_PEER_PORT2=8292
+++ MASTER_PEER_PORT3=8293
+++ MASTER_PEER_PORT4=8294
+++ MASTER_PEER_PORT5=8295
+++ MASTER_PEER_PORT6=8296
+++ WORKER1_PORT=8262
+++ WORKER2_PORT=8263
+++ WORKER3_PORT=8264
+++ WORKER4_PORT=18262
+++ WORKER5_PORT=18263
+++ WORKER1_NAME=worker1
+++ WORKER2_NAME=worker2
+++ SOURCE_ID1=mysql-replica-01
+++ SOURCE_ID2=mysql-replica-02
+++ RESET_MASTER=true
+++ VERBOSE=false
++ '[' true = true ']'
++ run_sql 'RESET MASTER' 3306 123456
++ run_sql 'RESET MASTER' 3307 123456
+ WORK_DIR=/tmp/dm_test/print_status
+ cleanup_data print_status
+ rm -rf /tmp/dm_test/print_status
+ mkdir /tmp/dm_test/print_status
+ for target_db in '"$@"'
+ run_sql 'drop database if exists `print_status`' 4000 123456
+ run_sql 'drop database if exists dm_meta' 4000 123456
+ cleanup_process
++ ps aux
++ grep dm-master.test temp
++ wc -l
++ rm temp
+ dm_master_num=0
+ echo '0 dm-master alive'
0 dm-master alive
+ pkill -hup dm-master.test
+ true
++ ps aux
++ grep dm-worker.test temp
++ wc -l
++ rm temp
+ dm_worker_num=0
+ echo '0 dm-worker alive'
0 dm-worker alive
+ pkill -hup dm-worker.test
+ true
++ ps aux
++ grep dm-syncer.test temp
++ wc -l
++ rm temp
+ dm_syncer_num=0
+ echo '0 dm-syncer alive'
0 dm-syncer alive
+ pkill -hup dm-syncer.test
+ true
+ wait_process_exit dm-master.test
process dm-master.test already exit
+ wait_process_exit dm-worker.test
process dm-worker.test already exit
+ wait_process_exit dm-syncer.test
process dm-syncer.test already exit
+ run
+ run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.prepare.sql 127.0.0.1 3306 123456
+ inject_points=("github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)" "github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)" "github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)")
++ join_string ';' 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1)' 'github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100)' 'github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)'
++ local 'IFS=;'
++ shift
++ echo 'github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)'
+ export 'GO_FAILPOINTS=github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)'
+ GO_FAILPOINTS='github.com/pingcap/tiflow/dm/worker/PrintStatusCheckSeconds=return(1);github.com/pingcap/tiflow/dm/loader/LoadDataSlowDown=sleep(100);github.com/pingcap/tiflow/dm/syncer/ProcessBinlogSlowDown=sleep(4)'
+ cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-worker1.toml /tmp/dm_test/print_status/dm-worker1.toml
+ sed -i s%placeholder%/tmp/dm_test/print_status/relay_by_worker%g /tmp/dm_test/print_status/dm-worker1.toml
+ run_dm_master /tmp/dm_test/print_status/master 8261 /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml
[Sun Apr 28 20:19:56 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-master.toml >>>>>>
+ check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_master_online 127.0.0.1:8261
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
[Sun Apr 28 20:19:56 CST 2024] <<<<<< start DM-010  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta"
wait process dm-master.test exit...
check diff failed 10-th time, retry later
rpc addr 127.0.0.1:8261 is alive
+ run_dm_worker /tmp/dm_test/print_status/worker1 8262 /tmp/dm_test/print_status/dm-worker1.toml
[Sun Apr 28 20:19:57 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /tmp/dm_test/print_status/dm-worker1.toml >>>>>>
+ check_rpc_alive /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/../bin/check_worker_online 127.0.0.1:8262
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
dmctl test cmd: "query-status test"
got=1 expected=1
wait process dm-master.test exit...
check diff failed 1-th time, retry later
rpc addr 127.0.0.1:8262 is alive
+ cp /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/source1.yaml /tmp/dm_test/print_status/source1.yaml
+ sed -i '/relay-binlog-name/i\relay-dir: /tmp/dm_test/print_status/worker1/relay_log' /tmp/dm_test/print_status/source1.yaml
+ dmctl_operate_source create /tmp/dm_test/print_status/source1.yaml mysql-replica-01
+ operate=create
+ source_toml=/tmp/dm_test/print_status/source1.yaml
+ source_id=mysql-replica-01
+ run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'operate-source create /tmp/dm_test/print_status/source1.yaml' '"result": true' 2 '"source": "mysql-replica-01"' 1
dmctl test cmd: "operate-source create /tmp/dm_test/print_status/source1.yaml"
wait process dm-master.test exit...
+ dmctl_start_task_standalone
+ '[' 0 -ge 2 ']'
+ remove_meta=
+ '[' 0 -ge 1 ']'
+ task_conf=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml
+ run_dm_ctl /tmp/dm_test/print_status 127.0.0.1:8261 'start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml ' '"result": true' 2 '"source": "mysql-replica-01"' 1
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/dm-task.yaml "
check diff failed at last
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:19:59 CST 2024] <<<<<< finish DM-119 pessimistic >>>>>>
[Sun Apr 28 20:19:59 CST 2024] <<<<<< start DM-119 optimistic >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl4/conf/double-source-optimistic.yaml --remove-meta"
wait process dm-master.test exit...
+ check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml
check diff successfully
+ ls /tmp/dm_test/print_status/relay_by_worker/worker1/57b2c0eb-0559-11ef-b869-b26a1028f92e.000001 /tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index
/tmp/dm_test/print_status/relay_by_worker/worker1/server-uuid.index

/tmp/dm_test/print_status/relay_by_worker/worker1/57b2c0eb-0559-11ef-b869-b26a1028f92e.000001:
dm-it-ae58409b-5fcc-4bb0-8272-b2488d0a0227-1s3sk-bfvwp-bin.000001
relay.meta
+ run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/data/db.increment.sql 127.0.0.1 3306 123456
+ check_sync_diff /tmp/dm_test/print_status /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/print_status/conf/diff_config.toml
check diff failed 1-th time, retry later
check diff successfully
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:20:00 CST 2024] <<<<<< finish DM-010  >>>>>>
dmctl test cmd: "query-status test"
got=2 expected=2
check diff failed 1-th time, retry later
wait process dm-master.test exit...
[Sun Apr 28 20:20:01 CST 2024] <<<<<< start DM-011  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-master.test exit...
check diff successfully
+ check_log_contains /tmp/dm_test/print_status/worker1/log/dm-worker.log 'enable safe-mode because of task initialization.*duration=1m0s'
+ cleanup_process
++ ps aux
++ grep dm-master.test temp
++ wc -l
++ rm temp
+ dm_master_num=1
+ echo '1 dm-master alive'
1 dm-master alive
+ pkill -hup dm-master.test
++ ps aux
++ grep dm-worker.test temp
++ wc -l
++ rm temp
+ dm_worker_num=1
+ echo '1 dm-worker alive'
1 dm-worker alive
+ pkill -hup dm-worker.test
++ ps aux
++ grep dm-syncer.test temp
++ wc -l
++ rm temp
+ dm_syncer_num=0
+ echo '0 dm-syncer alive'
0 dm-syncer alive
+ pkill -hup dm-syncer.test
+ true
+ wait_process_exit dm-master.test
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "stop-task test"
wait process dm-master.test exit...
wait process dm-master.test exit...
check diff failed 2-th time, retry later
[Sun Apr 28 20:20:03 CST 2024] <<<<<< finish DM-011  >>>>>>
wait process dm-master.test exit...
wait process dm-master.test exit...
process dm-master.test already exit
+ wait_process_exit dm-worker.test
[Sun Apr 28 20:20:04 CST 2024] <<<<<< start DM-012  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/double-source-pessimistic.yaml --remove-meta"
wait process dm-master.test exit...
check diff failed 3-th time, retry later
wait process dm-worker.test exit...
dmctl test cmd: "query-status test"
got=2 expected=2
dmctl test cmd: "stop-task test"
wait process dm-master.test exit...
[Sun Apr 28 20:20:06 CST 2024] <<<<<< finish DM-012  >>>>>>
wait process dm-worker.test exit...
wait process dm-master.test exit...
check diff failed 4-th time, retry later
[Sun Apr 28 20:20:07 CST 2024] <<<<<< start DM-013  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta"
wait process dm-worker.test exit...
process dm-worker.test already exit
+ wait_process_exit dm-syncer.test
process dm-syncer.test already exit
+ check_print_status
+ i=0
+ '[' 0 -lt 3 ']'
++ grep 'dm-worker exit' /tmp/dm_test/print_status/worker1/log/dm-worker.log
+ exit_log='[2024/04/28 20:20:05.311 +08:00] [INFO] [main.go:93] ["dm-worker exit"]'
+ '[' '[2024/04/28 20:20:05.311 +08:00] [INFO] [main.go:93] ["dm-worker exit"]' == 'not found' ']'
+ break
+ '[' 0 -ge 3 ']'
+ echo 'checking print status'
checking print status
+ dump_status_file=/tmp/dm_test/print_status/worker1/log/dump_status.log
+ grep -o 'progress status of dumpling' /tmp/dm_test/print_status/worker1/log/dm-worker.log
++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log
++ awk '{print $1}'
+ dump_status_count=1
+ '[' 1 -ge 1 ']'
+ grep -o '\[bps=0' /tmp/dm_test/print_status/worker1/log/dm-worker.log
+ true
++ wc -l /tmp/dm_test/print_status/worker1/log/dump_status.log
++ awk '{print $1}'
+ dump_status_count=0
+ '[' 0 -eq 0 ']'
+ status_file=/tmp/dm_test/print_status/worker1/log/loader_status.log
+ grep -oP '\[unit=lightning-load\] \[IsCanceled=false\] \[finished_bytes=59674\] \[total_bytes=59674\] \[progress=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log
++ wc -l /tmp/dm_test/print_status/worker1/log/loader_status.log
++ awk '{print $1}'
+ status_count=1
+ '[' 1 -eq 1 ']'
+ grep 'current speed (bytes / seconds)' /tmp/dm_test/print_status/worker1/log/dm-worker.log
+ grep -vq '"current speed (bytes / seconds)"=0'
+ echo 'check load unit print status success'
check load unit print status success
+ status_file2=/tmp/dm_test/print_status/worker1/log/syncer_status.log
+ grep -oP '\[total_rows=[0-9]+\] \[total_rps=[0-9]+\] \[rps=[0-9]+\] \[master_position=.*\]' /tmp/dm_test/print_status/worker1/log/dm-worker.log
++ wc -l /tmp/dm_test/print_status/worker1/log/syncer_status.log
++ awk '{print $1}'
+ status_count2=7
+ '[' 7 -ge 1 ']'
+ echo 'check sync unit print status success'
check sync unit print status success
++ date
+ echo '[Sun Apr 28 20:20:07 CST 2024] <<<<<< test case print_status success! >>>>>>'
[Sun Apr 28 20:20:07 CST 2024] <<<<<< test case print_status success! >>>>>>
start running case: [http_apis] script: [/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh]
Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/run.sh...
Verbose mode = false
0 dm-master alive
0 dm-worker alive
0 dm-syncer alive
process dm-master.test already exit
process dm-worker.test already exit
process dm-syncer.test already exit
[Sun Apr 28 20:20:08 CST 2024] <<<<<< START DM-MASTER on port 8261, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-master.toml >>>>>>
wait for rpc addr 127.0.0.1:8261 alive the 1-th time
wait process dm-master.test exit...
dmctl test cmd: "query-status test"
got=1 expected=1
check diff successfully
dmctl test cmd: "stop-task test"
wait process dm-master.test exit...
[Sun Apr 28 20:20:09 CST 2024] <<<<<< finish DM-013  >>>>>>
check diff failed 5-th time, retry later
wait process dm-master.test exit...
[Sun Apr 28 20:20:10 CST 2024] <<<<<< start DM-014  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-optimistic.yaml --remove-meta"
rpc addr 127.0.0.1:8261 is alive
[Sun Apr 28 20:20:10 CST 2024] <<<<<< START DM-WORKER on port 8262, config: /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/http_apis/conf/dm-worker1.toml >>>>>>
wait for rpc addr 127.0.0.1:8262 alive the 1-th time
Agent dm-it-818dc11f-aff0-4fa7-9472-cd09c9ab8d64-q18p4-njl57 is provisioned from template dm-it-818dc11f-aff0-4fa7-9472-cd09c9ab8d64-q18p4
---
apiVersion: "v1"
kind: "Pod"
metadata:
  annotations:
    buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/293/"
    runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/293/"
  labels:
    jenkins/jenkins-jenkins-agent: "true"
    jenkins/label-digest: "93004f46eb037b86086bae7656ea0f5579c81bbf"
    jenkins/label: "dm-it-818dc11f-aff0-4fa7-9472-cd09c9ab8d64"
  name: "dm-it-818dc11f-aff0-4fa7-9472-cd09c9ab8d64-q18p4-njl57"
  namespace: "jenkins-tiflow"
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: "kubernetes.io/arch"
            operator: "In"
            values:
            - "amd64"
  containers:
  - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest"
    name: "golang"
    resources:
      limits:
        memory: "16Gi"
        cpu: "6"
      requests:
        memory: "12Gi"
        cpu: "4"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - args:
    - "--ssl=ON"
    - "--log-bin"
    - "--binlog-format=ROW"
    - "--enforce-gtid-consistency=ON"
    - "--gtid-mode=ON"
    - "--server-id=1"
    - "--default-authentication-plugin=mysql_native_password"
    env:
    - name: "MYSQL_ROOT_PASSWORD"
      value: "123456"
    image: "hub.pingcap.net/jenkins/mysql:5.7"
    name: "mysql1"
    resources:
      limits:
        memory: "4Gi"
        cpu: "2"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - args:
    - "--ssl=ON"
    - "--log-bin"
    - "--binlog-format=ROW"
    - "--enforce-gtid-consistency=ON"
    - "--gtid-mode=ON"
    - "--server-id=1"
    - "--default-authentication-plugin=mysql_native_password"
    env:
    - name: "MYSQL_ROOT_PASSWORD"
      value: "123456"
    - name: "MYSQL_TCP_PORT"
      value: "3307"
    image: "registry-mirror.pingcap.net/library/mysql:8.0.21"
    name: "mysql2"
    resources:
      limits:
        memory: "4Gi"
        cpu: "2"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - image: "hub.pingcap.net/jenkins/network-multitool"
    name: "net-tool"
    resources:
      limits:
        memory: "128Mi"
        cpu: "100m"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - image: "hub.pingcap.net/jenkins/python3-requests:latest"
    name: "report"
    resources:
      limits:
        memory: "256Mi"
        cpu: "100m"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - env:
    - name: "JENKINS_SECRET"
      value: "********"
    - name: "JENKINS_TUNNEL"
      value: "jenkins-agent.apps.svc.cluster.local:50000"
    - name: "JENKINS_AGENT_NAME"
      value: "dm-it-818dc11f-aff0-4fa7-9472-cd09c9ab8d64-q18p4-njl57"
    - name: "JENKINS_NAME"
      value: "dm-it-818dc11f-aff0-4fa7-9472-cd09c9ab8d64-q18p4-njl57"
    - name: "JENKINS_AGENT_WORKDIR"
      value: "/home/jenkins/agent"
    - name: "JENKINS_URL"
      value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/"
    image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2"
    name: "jnlp"
    resources:
      requests:
        memory: "256Mi"
        cpu: "100m"
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  restartPolicy: "Never"
  securityContext:
    fsGroup: 1000
  volumes:
  - emptyDir:
      medium: ""
    name: "workspace-volume"

Running on dm-it-818dc11f-aff0-4fa7-9472-cd09c9ab8d64-q18p4-njl57 in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test
[Pipeline] {
wait process dm-master.test exit...
[Pipeline] checkout
The recommended git tool is: git
rpc addr 127.0.0.1:8262 is alive
source-id: mysql-replica-01\nflavor: ''\nenable-gtid: true\nenable-relay: true\nrelay-dir: /tmp/dm_test/http_apis/worker1/relay_log\nrelay-binlog-name: ''\nrelay-binlog-gtid: ''\nfrom:\n host: 127.0.0.1\n user: root\n password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=\n port: 3306\n
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   470  100   166  100   304   1281   2347 --:--:-- --:--:-- --:--:--  2338
100   470  100   166  100   304   1281   2346 --:--:-- --:--:-- --:--:--  2338
start task and check stage
---\nname: test\ntask-mode: all\nis-sharding: false\nmeta-schema: \"dm_meta\"\n# enable-heartbeat: true\n\ntarget-database:\n host: \"127.0.0.1\"\n port: 4000\n user: \"root\"\n password: \"\"\n\nmysql-instances:\n - source-id: \"mysql-replica-01\"\n block-allow-list: \"instance\"\n mydumper-config-name: \"global\"\n loader-config-name: \"global\"\n syncer-config-name: \"global\"\n\nblock-allow-list:\n instance:\n do-dbs: [\"http_apis\"]\n\nmydumpers:\n global:\n threads: 4\n chunk-filesize: 0\n skip-tz-utc: true\n statement-size: 100\n extra-args: \"\"\n\nloaders:\n global:\n pool-size: 16\n dir: \"./dumped_data\"\n\nsyncers:\n global:\n worker-count: 16\n batch: 100\n
dmctl test cmd: "query-status test"
got=1 expected=1
run tidb sql failed 1-th time, retry later
check diff failed 6-th time, retry later
HTTP 127.0.0.1:8261/apis/v1alpha1/status/test-task is alive
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
wait process dm-master.test exit...
No credentials specified
Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1b2aeb11; decorates RemoteLauncher[hudson.remoting.Channel@71a3a6ea:JNLP4-connect connection from 10.233.105.138/10.233.105.138:51038] will be ignored (a typical symptom is the Git executable not being run inside a designated container)
Cloning the remote Git repository
Using shallow clone with depth 1
Cloning repository https://github.com/PingCAP-QE/ci.git
 > git init /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test # timeout=10
Fetching upstream changes from https://github.com/PingCAP-QE/ci.git
 > git --version # timeout=10
 > git --version # 'git version 2.39.2'
 > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   742    0     0  100   742      0    740  0:00:01  0:00:01 --:--:--   741
100  1302  100   560  100   742    519    688  0:00:01  0:00:01 --:--:--   688
{
  "result": true,
  "msg": "",
  "sources": [
    {
      "result": false,
      "msg": "[code=38032:class=dm-master:scope=internal:level=high], Message: some error occurs in dm-worker: ErrCode:11121 ErrClass:\"functional\" ErrScope:\"internal\" ErrLevel:\"high\" Message:\"fail to restart streamer controller: no relay pos match gtid 57b2c0eb-0559-11ef-b869-b26a1028f92e:1-103\" , Workaround: Please execute `query-status` to check status.",
      "source": "mysql-replica-01",
      "worker": "worker1"
    }
  ],
  "checkResult": "pre-check is passed. "
}/tmp/dm_test/http_apis/start-task.log dosen't contain 2 "result": true
curl: (7) Failed connect to 127.0.0.1:8361; Connection refused
curl: (7) Failed connect to 127.0.0.1:8461; Connection refused
curl: (7) Failed connect to 127.0.0.1:8561; Connection refused
curl: (7) Failed connect to 127.0.0.1:8661; Connection refused
curl: (7) Failed connect to 127.0.0.1:8761; Connection refused
curl: (7) Failed connect to 127.0.0.1:8263; Connection refused
curl: (7) Failed connect to 127.0.0.1:8264; Connection refused
curl: (7) Failed connect to 127.0.0.1:18262; Connection refused
curl: (7) Failed connect to 127.0.0.1:18263; Connection refused
make: *** [dm_integration_test_in_group] Error 1
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
Post stage
[Pipeline] sh
wait process dm-master.test exit...
check diff failed 7-th time, retry later
+ ls /tmp/dm_test
cov.print_status.dmctl.1714306798.7110.out
cov.print_status.dmctl.1714306798.7153.out
cov.print_status.master.out
cov.print_status.worker.8262.1714306797.out
cov.start_task.dmctl.1714306704.695.out
cov.start_task.dmctl.1714306708.771.out
cov.start_task.dmctl.1714306710.830.out
cov.start_task.dmctl.1714306710.869.out
cov.start_task.dmctl.1714306710.910.out
cov.start_task.dmctl.1714306711.1028.out
cov.start_task.dmctl.1714306711.1072.out
cov.start_task.dmctl.1714306712.1128.out
cov.start_task.dmctl.1714306713.1166.out
cov.start_task.dmctl.1714306713.1212.out
cov.start_task.dmctl.1714306715.1261.out
cov.start_task.dmctl.1714306715.1307.out
cov.start_task.dmctl.1714306716.1348.out
cov.start_task.dmctl.1714306716.1389.out
cov.start_task.dmctl.1714306725.2501.out
cov.start_task.dmctl.1714306725.2542.out
cov.start_task.dmctl.1714306730.2928.out
cov.start_task.dmctl.1714306731.2968.out
cov.start_task.dmctl.1714306742.4539.out
cov.start_task.dmctl.1714306742.4588.out
cov.start_task.dmctl.1714306743.4626.out
cov.start_task.dmctl.1714306755.4927.out
cov.start_task.dmctl.1714306764.5677.out
cov.start_task.dmctl.1714306766.5727.out
cov.start_task.dmctl.1714306766.5766.out
cov.start_task.dmctl.1714306778.6071.out
cov.start_task.dmctl.1714306787.6521.out
cov.start_task.dmctl.1714306787.6564.out
cov.start_task.dmctl.1714306787.6612.out
cov.start_task.dmctl.1714306788.6668.out
cov.start_task.master.out
cov.start_task.worker.8262.1714306703.out
cov.start_task.worker.8262.1714306724.out
cov.start_task.worker.8262.1714306741.out
cov.start_task.worker.8262.1714306749.out
cov.start_task.worker.8262.1714306763.out
cov.start_task.worker.8262.1714306772.out
cov.start_task.worker.8262.1714306786.out
cov.start_task.worker.8263.1714306710.out
downstream
goroutines
http_apis
print_status
sql_res.http_apis.txt
sql_res.print_status.txt
sql_res.start_task.txt
start_task
tidb.toml
++ find /tmp/dm_test/ -type f -name '*.log'
+ tar -cvzf log-G10.tar.gz /tmp/dm_test/goroutines/stack/log/worker-8263.log /tmp/dm_test/goroutines/stack/log/worker-18263.log /tmp/dm_test/goroutines/stack/log/master-8661.log /tmp/dm_test/goroutines/stack/log/master-8361.log /tmp/dm_test/goroutines/stack/log/worker-8264.log /tmp/dm_test/goroutines/stack/log/master-8561.log /tmp/dm_test/goroutines/stack/log/master-8261.log /tmp/dm_test/goroutines/stack/log/master-8761.log /tmp/dm_test/goroutines/stack/log/worker-8262.log /tmp/dm_test/goroutines/stack/log/master-8461.log /tmp/dm_test/goroutines/stack/log/worker-18262.log /tmp/dm_test/print_status/master/log/stdout.log /tmp/dm_test/print_status/master/log/dm-master.log /tmp/dm_test/print_status/sync_diff_stdout.log /tmp/dm_test/print_status/worker1/log/dump_status.log /tmp/dm_test/print_status/worker1/log/syncer_status.log /tmp/dm_test/print_status/worker1/log/stdout.log /tmp/dm_test/print_status/worker1/log/loader_status.log /tmp/dm_test/print_status/worker1/log/dm-worker.log /tmp/dm_test/print_status/dmctl.1714306798.log /tmp/dm_test/http_apis/start-task.log /tmp/dm_test/http_apis/master/log/stdout.log /tmp/dm_test/http_apis/master/log/dm-master.log /tmp/dm_test/http_apis/worker1/log/stdout.log /tmp/dm_test/http_apis/worker1/log/dm-worker.log /tmp/dm_test/http_apis/create-source.log /tmp/dm_test/downstream/tidb/log/tidb.log /tmp/dm_test/start_task/0/master/log/stdout.log /tmp/dm_test/start_task/0/master/log/dm-master.log /tmp/dm_test/start_task/0/dmctl.1714306743.log /tmp/dm_test/start_task/0/dmctl.1714306755.log /tmp/dm_test/start_task/0/dmctl.1714306742.log /tmp/dm_test/start_task/0/sync_diff_stdout.log /tmp/dm_test/start_task/0/worker1/log/stdout.log /tmp/dm_test/start_task/0/worker1/log/dm-worker.log /tmp/dm_test/start_task/1/master/log/stdout.log /tmp/dm_test/start_task/1/master/log/dm-master.log /tmp/dm_test/start_task/1/dmctl.1714306787.log /tmp/dm_test/start_task/1/sync_diff_stdout.log /tmp/dm_test/start_task/1/dmctl.1714306788.log /tmp/dm_test/start_task/1/worker1/log/stdout.log /tmp/dm_test/start_task/1/worker1/log/dm-worker.log
tar: Removing leading `/' from member names
/tmp/dm_test/goroutines/stack/log/worker-8263.log
/tmp/dm_test/goroutines/stack/log/worker-18263.log
/tmp/dm_test/goroutines/stack/log/master-8661.log
/tmp/dm_test/goroutines/stack/log/master-8361.log
/tmp/dm_test/goroutines/stack/log/worker-8264.log
/tmp/dm_test/goroutines/stack/log/master-8561.log
/tmp/dm_test/goroutines/stack/log/master-8261.log
/tmp/dm_test/goroutines/stack/log/master-8761.log
/tmp/dm_test/goroutines/stack/log/worker-8262.log
/tmp/dm_test/goroutines/stack/log/master-8461.log
/tmp/dm_test/goroutines/stack/log/worker-18262.log
/tmp/dm_test/print_status/master/log/stdout.log
/tmp/dm_test/print_status/master/log/dm-master.log
/tmp/dm_test/print_status/sync_diff_stdout.log
/tmp/dm_test/print_status/worker1/log/dump_status.log
/tmp/dm_test/print_status/worker1/log/syncer_status.log
/tmp/dm_test/print_status/worker1/log/stdout.log
/tmp/dm_test/print_status/worker1/log/loader_status.log
/tmp/dm_test/print_status/worker1/log/dm-worker.log
/tmp/dm_test/print_status/dmctl.1714306798.log
/tmp/dm_test/http_apis/start-task.log
/tmp/dm_test/http_apis/master/log/stdout.log
/tmp/dm_test/http_apis/master/log/dm-master.log
/tmp/dm_test/http_apis/worker1/log/stdout.log
/tmp/dm_test/http_apis/worker1/log/dm-worker.log
/tmp/dm_test/http_apis/create-source.log
/tmp/dm_test/downstream/tidb/log/tidb.log
/tmp/dm_test/start_task/0/master/log/stdout.log
/tmp/dm_test/start_task/0/master/log/dm-master.log
/tmp/dm_test/start_task/0/dmctl.1714306743.log
/tmp/dm_test/start_task/0/dmctl.1714306755.log
/tmp/dm_test/start_task/0/dmctl.1714306742.log
/tmp/dm_test/start_task/0/sync_diff_stdout.log
/tmp/dm_test/start_task/0/worker1/log/stdout.log
/tmp/dm_test/start_task/0/worker1/log/dm-worker.log
/tmp/dm_test/start_task/1/master/log/stdout.log
/tmp/dm_test/start_task/1/master/log/dm-master.log
/tmp/dm_test/start_task/1/dmctl.1714306787.log
/tmp/dm_test/start_task/1/sync_diff_stdout.log
/tmp/dm_test/start_task/1/dmctl.1714306788.log
/tmp/dm_test/start_task/1/worker1/log/stdout.log
/tmp/dm_test/start_task/1/worker1/log/dm-worker.log
+ ls -alh log-G10.tar.gz
-rw-r--r-- 1 jenkins jenkins 435K Apr 28 20:20 log-G10.tar.gz
[Pipeline] archiveArtifacts
Archiving artifacts
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
dmctl test cmd: "stop-task test"
[Sun Apr 28 20:20:13 CST 2024] <<<<<< finish DM-014  >>>>>>
[Pipeline] // timeout
[Pipeline] }
wait process dm-master.test exit...
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Sun Apr 28 20:20:14 CST 2024] <<<<<< start DM-015  >>>>>>
dmctl test cmd: "start-task /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test/tiflow/dm/tests/shardddl1_1/conf/single-source-pessimistic.yaml --remove-meta"
[Pipeline] // withEnv
[Pipeline] }
wait process dm-master.test exit...
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G10'
Sending interrupt signal to process
Killing processes
kill finished with exit code 0
Sending interrupt signal to process
Killing processes
Agent dm-it-6b9dd44b-75ac-4051-9434-78774ab6e71e-81fqc-rhvfm is provisioned from template dm-it-6b9dd44b-75ac-4051-9434-78774ab6e71e-81fqc
---
apiVersion: "v1"
kind: "Pod"
metadata:
  annotations:
    buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/293/"
    runUrl: "job/pingcap/job/tiflow/job/release-7.5/job/pull_dm_integration_test/293/"
  labels:
    jenkins/jenkins-jenkins-agent: "true"
    jenkins/label-digest: "7aeb76aa474ec7e475983c6530b28c5d9d9fdde1"
    jenkins/label: "dm-it-6b9dd44b-75ac-4051-9434-78774ab6e71e"
  name: "dm-it-6b9dd44b-75ac-4051-9434-78774ab6e71e-81fqc-rhvfm"
  namespace: "jenkins-tiflow"
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: "kubernetes.io/arch"
            operator: "In"
            values:
            - "amd64"
  containers:
  - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest"
    name: "golang"
    resources:
      limits:
        memory: "16Gi"
        cpu: "6"
      requests:
        memory: "12Gi"
        cpu: "4"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - args:
    - "--ssl=ON"
    - "--log-bin"
    - "--binlog-format=ROW"
    - "--enforce-gtid-consistency=ON"
    - "--gtid-mode=ON"
    - "--server-id=1"
    - "--default-authentication-plugin=mysql_native_password"
    env:
    - name: "MYSQL_ROOT_PASSWORD"
      value: "123456"
    image: "hub.pingcap.net/jenkins/mysql:5.7"
    name: "mysql1"
    resources:
      limits:
        memory: "4Gi"
        cpu: "2"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - args:
    - "--ssl=ON"
    - "--log-bin"
    - "--binlog-format=ROW"
    - "--enforce-gtid-consistency=ON"
    - "--gtid-mode=ON"
    - "--server-id=1"
    - "--default-authentication-plugin=mysql_native_password"
    env:
    - name: "MYSQL_ROOT_PASSWORD"
      value: "123456"
    - name: "MYSQL_TCP_PORT"
      value: "3307"
    image: "registry-mirror.pingcap.net/library/mysql:8.0.21"
    name: "mysql2"
    resources:
      limits:
        memory: "4Gi"
        cpu: "2"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - image: "hub.pingcap.net/jenkins/network-multitool"
    name: "net-tool"
    resources:
      limits:
        memory: "128Mi"
        cpu: "100m"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - image: "hub.pingcap.net/jenkins/python3-requests:latest"
    name: "report"
    resources:
      limits:
        memory: "256Mi"
        cpu: "100m"
    tty: true
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  - env:
    - name: "JENKINS_SECRET"
      value: "********"
    - name: "JENKINS_TUNNEL"
      value: "jenkins-agent.apps.svc.cluster.local:50000"
    - name: "JENKINS_AGENT_NAME"
      value: "dm-it-6b9dd44b-75ac-4051-9434-78774ab6e71e-81fqc-rhvfm"
    - name: "JENKINS_NAME"
      value: "dm-it-6b9dd44b-75ac-4051-9434-78774ab6e71e-81fqc-rhvfm"
    - name: "JENKINS_AGENT_WORKDIR"
      value: "/home/jenkins/agent"
    - name: "JENKINS_URL"
      value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/"
    image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2"
    name: "jnlp"
    resources:
      requests:
        memory: "256Mi"
        cpu: "100m"
    volumeMounts:
    - mountPath: "/home/jenkins/agent"
      name: "workspace-volume"
      readOnly: false
  restartPolicy: "Never"
  securityContext:
    fsGroup: 1000
  volumes:
  - emptyDir:
      medium: ""
    name: "workspace-volume"

kill finished with exit code 0
Sending interrupt signal to process
Killing processes
Running on dm-it-6b9dd44b-75ac-4051-9434-78774ab6e71e-81fqc-rhvfm in /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_dm_integration_test
check diff failed 8-th time, retry later
kill finished with exit code 0
Sending interrupt signal to process
Killing processes
make: *** [dm_integration_test_in_group] Terminated
script returned exit code 143
kill finished with exit code 0
dmctl test cmd: "query-status test"
make: *** [dm_integration_test_in_group] Terminated
script returned exit code 143
[Pipeline] // node
[Pipeline] }
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] // cache
make: *** [dm_integration_test_in_group] Terminated
script returned exit code 143
[Pipeline] // cache
[Pipeline] // node
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // withEnv
[Pipeline] // podTemplate
[Pipeline] // dir
[Pipeline] // dir
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // cache
[Pipeline] }
[Pipeline] // stage
[Pipeline] // withEnv
[Pipeline] // withCredentials
[Pipeline] // withCredentials
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'TLS_GROUP'
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // stage
[Pipeline] // timeout
[Pipeline] // timeout
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G11'
[Pipeline] }
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // stage
[Pipeline] // stage
[Pipeline] }
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // container
[Pipeline] // container
[Pipeline] }
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // node
[Pipeline] // node
[Pipeline] }
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // stage
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G01'
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G07'
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G08'
make: *** [dm_integration_test_in_group] Terminated
script returned exit code 143
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G00'
[Pipeline] // parallel
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
ERROR: script returned exit code 2
Finished: FAILURE