Console Output

Skipping 1,656 KB.. Full Log
[2024/04/28 11:56:03.297 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0019c61c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
check_changefeed_status 127.0.0.1:8300 cbd01616-6ac0-49b9-ac3d-278bdce56be1 normal last_error null
+ endpoint=127.0.0.1:8300
+ changefeed_id=cbd01616-6ac0-49b9-ac3d-278bdce56be1
+ expected_state=normal
+ field=last_error
+ error_pattern=null
++ curl 127.0.0.1:8300/api/v2/changefeeds/cbd01616-6ac0-49b9-ac3d-278bdce56be1/status
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100    86  100    86    0     0   1170      0 --:--:-- --:--:-- --:--:--  1178
+ info='{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
+ echo '{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}
++ echo '{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
+ [[ -z last_error ]]
++ echo '{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
++ jq -r .last_error.message
+ error_msg=null
+ [[ ! null =~ null ]]
run task successfully
check_changefeed_status 127.0.0.1:8300 cbd01616-6ac0-49b9-ac3d-278bdce56be1 normal last_warning null
+ endpoint=127.0.0.1:8300
+ changefeed_id=cbd01616-6ac0-49b9-ac3d-278bdce56be1
+ expected_state=normal
+ field=last_warning
+ error_pattern=null
++ curl 127.0.0.1:8300/api/v2/changefeeds/cbd01616-6ac0-49b9-ac3d-278bdce56be1/status
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
+ set +x
+ tso='449387315049791489
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449387315049791489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Sun Apr 28 11:56:03 CST 2024] <<<<<< START cdc server in savepoint case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ GO_FAILPOINTS=
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1556415566.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
*   Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100    86  100    86    0     0   1081      0 --:--:-- --:--:-- --:--:--  1075
+ info='{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
+ echo '{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}
++ echo '{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
+ [[ -z last_warning ]]
++ echo '{"state":"normal","resolved_ts":449387315112706072,"checkpoint_ts":449387315112706072}'
++ jq -r .last_warning.message
+ error_msg=null
+ [[ ! null =~ null ]]
run task successfully
table sink_hang.t1 not exists for 1-th check, retry later
table autorandom_test.table_a not exists for 3-th check, retry later
table partition_table.finish_mark not exists for 12-th check, retry later
check diff successfully
table sink_hang.t1 not exists for 2-th check, retry later
wait process cdc.test exit for 1-th time...
table partition_table.finish_mark not exists for 13-th check, retry later
table autorandom_test.table_a not exists for 4-th check, retry later
wait process cdc.test exit for 2-th time...
wait process cdc.test exit for 3-th time...
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
*   Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
> 
< HTTP/1.1 200 OK
< Date: Sun, 28 Apr 2024 03:56:06 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
< 
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/d4e5a468-01b6-44ab-9bd0-cce21781014e
	{"id":"d4e5a468-01b6-44ab-9bd0-cce21781014e","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d83487f5
	d4e5a468-01b6-44ab-9bd0-cce21781014e

/tidb/cdc/default/default/upstream/7362761710456744076
	{"id":7362761710456744076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/d4e5a468-01b6-44ab-9bd0-cce21781014e
	{"id":"d4e5a468-01b6-44ab-9bd0-cce21781014e","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d83487f5
	d4e5a468-01b6-44ab-9bd0-cce21781014e

/tidb/cdc/default/default/upstream/7362761710456744076
	{"id":7362761710456744076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/d4e5a468-01b6-44ab-9bd0-cce21781014e
	{"id":"d4e5a468-01b6-44ab-9bd0-cce21781014e","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d83487f5
	d4e5a468-01b6-44ab-9bd0-cce21781014e

/tidb/cdc/default/default/upstream/7362761710456744076
	{"id":7362761710456744076,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15618.out cli changefeed create --start-ts=449387315049791489 '--sink-uri=file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-25947?protocol=canal-json&enable-tidb-extension=true'
Create changefeed successfully!
ID: e08c47cc-26da-4ed6-92bf-620a32f9cadd
Info: {"upstream_id":7362761710456744076,"namespace":"default","id":"e08c47cc-26da-4ed6-92bf-620a32f9cadd","sink_uri":"file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-25947?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-28T11:56:06.830124045+08:00","start_ts":449387315049791489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-49-g234b58dc1","resolved_ts":449387315049791489,"checkpoint_ts":449387315049791489,"checkpoint_time":"2024-04-28 11:56:01.927"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
wait process cdc.test exit for 4-th time...
table partition_table.finish_mark not exists for 14-th check, retry later
wait process cdc.test exit for 5-th time...
table sink_hang.t1 not exists for 3-th check, retry later
table autorandom_test.table_a not exists for 5-th check, retry later
wait process cdc.test exit for 6-th time...
+ set +x
+ workdir=/tmp/tidb_cdc_test/savepoint
+ sink_uri='file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-25947?protocol=canal-json&enable-tidb-extension=true'
+ consumer_replica_config=
+ log_suffix=
++ pwd
+ pwd=/tmp/tidb_cdc_test/savepoint
++ date
+ echo '[Sun Apr 28 11:56:08 CST 2024] <<<<<< START storage consumer in savepoint case >>>>>>'
[Sun Apr 28 11:56:08 CST 2024] <<<<<< START storage consumer in savepoint case >>>>>>
+ cd /tmp/tidb_cdc_test/savepoint
+ '[' '' '!=' '' ']'
+ cd /tmp/tidb_cdc_test/savepoint
+ set +x
+ cdc_storage_consumer --log-file /tmp/tidb_cdc_test/savepoint/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-25947?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false'
table savepoint.finish_mark not exists for 1-th check, retry later
wait process cdc.test exit for 7-th time...
wait process cdc.test exit for 8-th time...
table partition_table.finish_mark exists
check diff successfully
wait process cdc.test exit for 9-th time...
table sink_hang.t1 not exists for 4-th check, retry later
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 10-th time...
wait process cdc.test exit for 2-th time...
table autorandom_test.table_a exists
check diff successfully
wait process cdc.test exit for 11-th time...
wait process cdc.test exit for 1-th time...
table savepoint.finish_mark not exists for 2-th check, retry later
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Sun Apr 28 11:56:11 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>>
wait process cdc.test exit for 12-th time...
wait process cdc.test exit for 2-th time...
table sink_hang.t1 not exists for 5-th check, retry later
wait process cdc.test exit for 13-th time...
[2024/04/28 11:56:11.718 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001b7a380/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
wait process cdc.test exit for 3-th time...
[2024/04/28 11:56:12.012 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0009a0000/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
wait process cdc.test exit for 14-th time...
wait process cdc.test exit for 4-th time...
cdc.test: no process found
wait process cdc.test exit for 5-th time...
process cdc.test already exit
[Sun Apr 28 11:56:12 CST 2024] <<<<<< run test case autorandom success! >>>>>>
wait process cdc.test exit for 15-th time...
table savepoint.finish_mark not exists for 3-th check, retry later
wait process cdc.test exit for 16-th time...
table sink_hang.t1 exists
table sink_hang.t2 exists
wait process cdc.test exit for 17-th time...
check diff failed 1-th time, retry later
table savepoint.finish_mark not exists for 4-th check, retry later
pass check, checkpoint tso not forward after 10s
run task successfully
wait process 8064 exit for 1-th time...
wait process 8064 exit for 2-th time...
wait process 8064 exit for 3-th time...
wait process 8064 exit for 4-th time...
/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8064) - No such process
wait process 8064 exit for 5-th time...
process 8064 already exit
[Sun Apr 28 11:56:08 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ GO_FAILPOINTS=
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.86578659.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
*   Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
*   Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
> 
< HTTP/1.1 200 OK
< Date: Sun, 28 Apr 2024 03:56:11 GMT
< Content-Type: text/plain; charset=utf-8
< Transfer-Encoding: chunked
< 
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='

*** owner info ***:



*** processors info ***:

changefeedID: default/ddl-only-block-related-table
{UpstreamID:7362761348588424331 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-4005?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-04-28 11:54:40.785452778 +0800 CST StartTs:449387293746135049 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004296510 State:normal Error:<nil> Warning:<nil> CreatorVersion:v7.5.1-49-g234b58dc1 Epoch:449387293772349445}
{CheckpointTs:449387298648227852 MinTableBarrierTs:449387317456797701 AdminJobType:noop}
span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/6ec80961-d6fc-41b4-b437-01eab4125549
	{"id":"6ec80961-d6fc-41b4-b437-01eab4125549","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d6e93585
	6ec80961-d6fc-41b4-b437-01eab4125549

/tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table
	{"upstream-id":7362761348588424331,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-4005?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-04-28T11:54:40.785452778+08:00","start-ts":449387293746135049,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-49-g234b58dc1","epoch":449387293772349445}

/tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table
	{"checkpoint-ts":449387298648227852,"min-table-barrier-ts":449387317456797701,"admin-job-type":0}

/tidb/cdc/default/default/task/position/6ec80961-d6fc-41b4-b437-01eab4125549/ddl-only-block-related-table
	{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}

/tidb/cdc/default/default/upstream/7362761348588424331
	{"id":7362761348588424331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '

*** owner info ***:



*** processors info ***:

changefeedID: default/ddl-only-block-related-table
{UpstreamID:7362761348588424331 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-4005?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-04-28 11:54:40.785452778 +0800 CST StartTs:449387293746135049 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004296510 State:normal Error:<nil> Warning:<nil> CreatorVersion:v7.5.1-49-g234b58dc1 Epoch:449387293772349445}
{CheckpointTs:449387298648227852 MinTableBarrierTs:449387317456797701 AdminJobType:noop}
span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/6ec80961-d6fc-41b4-b437-01eab4125549
	{"id":"6ec80961-d6fc-41b4-b437-01eab4125549","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d6e93585
	6ec80961-d6fc-41b4-b437-01eab4125549

/tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table
	{"upstream-id":7362761348588424331,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-4005?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-04-28T11:54:40.785452778+08:00","start-ts":449387293746135049,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-49-g234b58dc1","epoch":449387293772349445}

/tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table
	{"checkpoint-ts":449387298648227852,"min-table-barrier-ts":449387317456797701,"admin-job-type":0}

/tidb/cdc/default/default/task/position/6ec80961-d6fc-41b4-b437-01eab4125549/ddl-only-block-related-table
	{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}
+ grep -q 'failed to get info:'

/tidb/cdc/default/default/upstream/7362761348588424331
	{"id":7362761348588424331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '

*** owner info ***:



*** processors info ***:

changefeedID: default/ddl-only-block-related-table
{UpstreamID:7362761348588424331 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-4005?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-04-28 11:54:40.785452778 +0800 CST StartTs:449387293746135049 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004296510 State:normal Error:<nil> Warning:<nil> CreatorVersion:v7.5.1-49-g234b58dc1 Epoch:449387293772349445}
{CheckpointTs:449387298648227852 MinTableBarrierTs:449387317456797701 AdminJobType:noop}
span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:104,start_key:7480000000000000ff685f720000000000fa,end_key:7480000000000000ff685f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing
span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449387298648227852, checkpointTs: 449387298648227852, state: Preparing



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/6ec80961-d6fc-41b4-b437-01eab4125549
	{"id":"6ec80961-d6fc-41b4-b437-01eab4125549","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d6e93585
	6ec80961-d6fc-41b4-b437-01eab4125549

/tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table
	{"upstream-id":7362761348588424331,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-4005?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-04-28T11:54:40.785452778+08:00","start-ts":449387293746135049,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-49-g234b58dc1","epoch":449387293772349445}

/tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table
	{"checkpoint-ts":449387298648227852,"min-table-barrier-ts":449387317456797701,"admin-job-type":0}

/tidb/cdc/default/default/task/position/6ec80961-d6fc-41b4-b437-01eab4125549/ddl-only-block-related-table
	{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}

/tidb/cdc/default/default/upstream/7362761348588424331
	{"id":7362761348588424331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
check diff failed 1-th time, retry later
check diff successfully
check_ts_forward ddl-only-block-related-table
wait process cdc.test exit for 18-th time...
wait process cdc.test exit for 19-th time...
wait process cdc.test exit for 20-th time...
wait process cdc.test exit timeout
[2024/04/28 11:56:15.585 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001c861c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
check diff failed 2-th time, retry later
[2024/04/28 11:56:15.601 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0019c61c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
changefeed is working normally rts: 449387318243491840->449387318505373700 checkpoint: 449387318243491840->449387318505373700
run task successfully
table savepoint.finish_mark not exists for 5-th check, retry later
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Sun Apr 28 11:56:17 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>>
check diff failed 3-th time, retry later
table savepoint.finish_mark not exists for 6-th check, retry later
check diff failed 4-th time, retry later
table savepoint.finish_mark exists
check diff successfully
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Sun Apr 28 11:56:22 CST 2024] <<<<<< run test case savepoint success! >>>>>>
check diff successfully
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Sun Apr 28 11:56:23 CST 2024] <<<<<< run test case sink_hang success! >>>>>>
\033[0;36m<<< Run all test success >>>\033[0m
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_storage_test-293/tiflow-cdc already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: storage... <<=================
[Sun Apr 28 11:56:26 CST 2024] <<<<<< run test case generate_column success! >>>>>>
[2024/04/28 11:56:27.359 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc001c861c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
[2024/04/28 11:56:27.360 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0019c61c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
\033[0;36m<<< Run all test success >>>\033[0m
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_storage_test-293/tiflow-cdc already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: storage... <<=================
The 1 times to try to start tidb cluster...
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop
Starting Upstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time:  2024-04-03 10:04:14
Starting Downstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time:  2024-04-03 10:04:14
Verifying upstream PD is started...
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: storage... <<=================
+++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/run.sh
++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status
++ pwd
+ CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status
+ source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare
++ UP_TIDB_HOST=127.0.0.1
++ UP_TIDB_PORT=4000
++ UP_TIDB_OTHER_PORT=4001
++ UP_TIDB_STATUS=10080
++ UP_TIDB_OTHER_STATUS=10081
++ DOWN_TIDB_HOST=127.0.0.1
++ DOWN_TIDB_PORT=3306
++ DOWN_TIDB_STATUS=20080
++ TLS_TIDB_HOST=127.0.0.1
++ TLS_TIDB_PORT=3307
++ TLS_TIDB_STATUS=30080
++ UP_PD_HOST_1=127.0.0.1
++ UP_PD_PORT_1=2379
++ UP_PD_PEER_PORT_1=2380
++ UP_PD_HOST_2=127.0.0.1
++ UP_PD_PORT_2=2679
++ UP_PD_PEER_PORT_2=2680
++ UP_PD_HOST_3=127.0.0.1
++ UP_PD_PORT_3=2779
++ UP_PD_PEER_PORT_3=2780
++ DOWN_PD_HOST=127.0.0.1
++ DOWN_PD_PORT=2479
++ DOWN_PD_PEER_PORT=2480
++ TLS_PD_HOST=127.0.0.1
++ TLS_PD_PORT=2579
++ TLS_PD_PEER_PORT=2580
++ UP_TIKV_HOST_1=127.0.0.1
++ UP_TIKV_PORT_1=20160
++ UP_TIKV_STATUS_PORT_1=20181
++ UP_TIKV_HOST_2=127.0.0.1
++ UP_TIKV_PORT_2=20161
++ UP_TIKV_STATUS_PORT_2=20182
++ UP_TIKV_HOST_3=127.0.0.1
++ UP_TIKV_PORT_3=20162
++ UP_TIKV_STATUS_PORT_3=20183
++ DOWN_TIKV_HOST=127.0.0.1
++ DOWN_TIKV_PORT=21160
++ DOWN_TIKV_STATUS_PORT=21180
++ TLS_TIKV_HOST=127.0.0.1
++ TLS_TIKV_PORT=22160
++ TLS_TIKV_STATUS_PORT=22180
+++ cat /tmp/tidb_cdc_test/KAFKA_VERSION
+++ echo 2.4.1
++ KAFKA_VERSION=2.4.1
+ WORK_DIR=/tmp/tidb_cdc_test/synced_status
+ CDC_BINARY=cdc.test
+ SINK_TYPE=storage
+ CDC_COUNT=3
+ DB_COUNT=4
+ trap stop_tidb_cluster EXIT
+ run_normal_case_and_unavailable_pd conf/changefeed.toml
+ rm -rf /tmp/tidb_cdc_test/synced_status
+ mkdir -p /tmp/tidb_cdc_test/synced_status
+ start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status
The 1 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time:  2024-04-03 10:04:14
Starting Downstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time:  2024-04-03 10:04:14
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV 
Release Version:   7.5.2
Edition:           Community
Git Commit Hash:   46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time:    2024-04-25 09:30:46
Rust Version:      rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features:   pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile:           dist_release
Starting Downstream TiKV...
TiKV 
Release Version:   7.5.2
Edition:           Community
Git Commit Hash:   46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time:    2024-04-25 09:30:46
Rust Version:      rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features:   pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile:           dist_release
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
Post stage
[Pipeline] sh
+ ls /tmp/tidb_cdc_test/
big_txn
cov.big_txn.cli.2610.out
sql_res.big_txn.txt
++ find /tmp/tidb_cdc_test/ -type f -name '*.log'
+ tar -cvzf log-G14.tar.gz /tmp/tidb_cdc_test/big_txn/cdc.log /tmp/tidb_cdc_test/big_txn/tikv1.log /tmp/tidb_cdc_test/big_txn/down_pd.log /tmp/tidb_cdc_test/big_txn/pd1/region-meta/000001.log /tmp/tidb_cdc_test/big_txn/pd1/hot-region/000001.log /tmp/tidb_cdc_test/big_txn/stdout.log /tmp/tidb_cdc_test/big_txn/tikv_down/db/000014.log /tmp/tidb_cdc_test/big_txn/tikv_down/db/000018.log /tmp/tidb_cdc_test/big_txn/tikv_down/db/000005.log /tmp/tidb_cdc_test/big_txn/tikv_down/db/000026.log /tmp/tidb_cdc_test/big_txn/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/big_txn/tidb_down.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000007.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000016.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000006.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000011.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000008.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000005.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000016.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000014.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000018.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000010.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000008.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000012.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/big_txn/tiflash/log/error.log /tmp/tidb_cdc_test/big_txn/tiflash/log/server.log /tmp/tidb_cdc_test/big_txn/tiflash/log/proxy.log /tmp/tidb_cdc_test/big_txn/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/big_txn/tikv3/db/000005.log /tmp/tidb_cdc_test/big_txn/tikv3/db/000026.log /tmp/tidb_cdc_test/big_txn/tidb.log /tmp/tidb_cdc_test/big_txn/tikv2.log /tmp/tidb_cdc_test/big_txn/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/big_txn/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/big_txn/cdc_storage_consumer.log /tmp/tidb_cdc_test/big_txn/tikv1/db/000005.log /tmp/tidb_cdc_test/big_txn/pd1.log /tmp/tidb_cdc_test/big_txn/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/big_txn/tidb_other.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000034.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000046.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000030.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000052.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000020.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000015.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000005.log /tmp/tidb_cdc_test/big_txn/tikv2/db/000042.log /tmp/tidb_cdc_test/big_txn/tikv_down-2024-04-28T11-54-13.571.log /tmp/tidb_cdc_test/big_txn/tidb-slow.log /tmp/tidb_cdc_test/big_txn/tikv3.log /tmp/tidb_cdc_test/big_txn/tikv2-2024-04-28T11-54-15.055.log /tmp/tidb_cdc_test/big_txn/sync_diff_inspector.log /tmp/tidb_cdc_test/big_txn/tikv_down.log
tar: Removing leading `/' from member names
/tmp/tidb_cdc_test/big_txn/cdc.log
/tmp/tidb_cdc_test/big_txn/tikv1.log
\033[0;36m<<< Run all test success >>>\033[0m
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_storage_test-293/tiflow-cdc already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
Post stage
[Pipeline] sh
/tmp/tidb_cdc_test/big_txn/down_pd.log
/tmp/tidb_cdc_test/big_txn/pd1/region-meta/000001.log
/tmp/tidb_cdc_test/big_txn/pd1/hot-region/000001.log
/tmp/tidb_cdc_test/big_txn/stdout.log
/tmp/tidb_cdc_test/big_txn/tikv_down/db/000014.log
Starting Upstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ ls /tmp/tidb_cdc_test/
cov.owner_resign.25132515.out
cov.owner_resign.28952897.out
cov.owner_resign.cli.2583.out
cov.processor_etcd_worker_delay.56105612.out
cov.sink_hang.94119413.out
owner_resign
processor_etcd_worker_delay
sink_hang
sql_res.owner_resign.txt
sql_res.processor_etcd_worker_delay.txt
sql_res.sink_hang.txt
++ find /tmp/tidb_cdc_test/ -type f -name '*.log'
+ tar -cvzf log-G16.tar.gz /tmp/tidb_cdc_test/owner_resign/tidb.log /tmp/tidb_cdc_test/owner_resign/cdc_storage_consumer.log /tmp/tidb_cdc_test/owner_resign/cdc1.log /tmp/tidb_cdc_test/owner_resign/stdout2.log /tmp/tidb_cdc_test/owner_resign/cdc2.log /tmp/tidb_cdc_test/owner_resign/stdout1.log /tmp/tidb_cdc_test/owner_resign/tikv_down.log /tmp/tidb_cdc_test/owner_resign/tikv3.log /tmp/tidb_cdc_test/owner_resign/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/owner_resign/pd1.log /tmp/tidb_cdc_test/owner_resign/tidb_other.log /tmp/tidb_cdc_test/owner_resign/sync_diff_inspector.log /tmp/tidb_cdc_test/owner_resign/tikv1.log /tmp/tidb_cdc_test/owner_resign/tidb_down.log /tmp/tidb_cdc_test/owner_resign/tikv2.log /tmp/tidb_cdc_test/owner_resign/tidb-slow.log /tmp/tidb_cdc_test/owner_resign/down_pd.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_storage_consumer.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv_down.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv3.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/stdout.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/pd1.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb_other.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/sync_diff_inspector.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv1.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb_down.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv2.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb-slow.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/down_pd.log /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log /tmp/tidb_cdc_test/sink_hang/tidb.log /tmp/tidb_cdc_test/sink_hang/cdc_storage_consumer.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/sink_hang/tikv_down/db/000005.log /tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log /tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/sink_hang/tikv_down.log /tmp/tidb_cdc_test/sink_hang/tikv3.log /tmp/tidb_cdc_test/sink_hang/stdout.log /tmp/tidb_cdc_test/sink_hang/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/sink_hang/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/sink_hang/pd1.log /tmp/tidb_cdc_test/sink_hang/tidb_other.log /tmp/tidb_cdc_test/sink_hang/sync_diff_inspector.log /tmp/tidb_cdc_test/sink_hang/tikv1.log /tmp/tidb_cdc_test/sink_hang/tikv3/db/000005.log /tmp/tidb_cdc_test/sink_hang/tidb_down.log /tmp/tidb_cdc_test/sink_hang/tikv2.log /tmp/tidb_cdc_test/sink_hang/pd1/region-meta/000001.log /tmp/tidb_cdc_test/sink_hang/pd1/hot-region/000001.log /tmp/tidb_cdc_test/sink_hang/tidb-slow.log /tmp/tidb_cdc_test/sink_hang/tikv2/db/000005.log /tmp/tidb_cdc_test/sink_hang/down_pd.log /tmp/tidb_cdc_test/sink_hang/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/sink_hang/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/sink_hang/cdc.log /tmp/tidb_cdc_test/sink_hang/tikv1/db/000005.log
tar: Removing leading `/' from member names
/tmp/tidb_cdc_test/owner_resign/tidb.log
/tmp/tidb_cdc_test/owner_resign/cdc_storage_consumer.log
/tmp/tidb_cdc_test/owner_resign/cdc1.log
/tmp/tidb_cdc_test/owner_resign/stdout2.log
/tmp/tidb_cdc_test/owner_resign/cdc2.log
/tmp/tidb_cdc_test/owner_resign/stdout1.log
/tmp/tidb_cdc_test/owner_resign/tikv_down.log
/tmp/tidb_cdc_test/big_txn/tikv_down/db/000018.log
/tmp/tidb_cdc_test/owner_resign/tikv3.log
/tmp/tidb_cdc_test/owner_resign/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/owner_resign/pd1.log
/tmp/tidb_cdc_test/owner_resign/tidb_other.log
/tmp/tidb_cdc_test/owner_resign/sync_diff_inspector.log
/tmp/tidb_cdc_test/owner_resign/tikv1.log
/tmp/tidb_cdc_test/owner_resign/tidb_down.log
/tmp/tidb_cdc_test/owner_resign/tikv2.log
/tmp/tidb_cdc_test/owner_resign/tidb-slow.log
/tmp/tidb_cdc_test/owner_resign/down_pd.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_storage_consumer.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv_down.log
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV 
Release Version:   7.5.2
Edition:           Community
Git Commit Hash:   46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time:    2024-04-25 09:30:46
Rust Version:      rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features:   pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile:           dist_release
Starting Downstream TiKV...
TiKV 
Release Version:   7.5.2
Edition:           Community
Git Commit Hash:   46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time:    2024-04-25 09:30:46
Rust Version:      rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features:   pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile:           dist_release
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv3.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/stdout.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/pd1.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb_other.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/sync_diff_inspector.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv1.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb_down.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tikv2.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/tidb-slow.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/down_pd.log
/tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log
/tmp/tidb_cdc_test/sink_hang/tidb.log
/tmp/tidb_cdc_test/sink_hang/cdc_storage_consumer.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0003/000002.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0001/000002.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0004/000002.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0000/000002.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0006/000002.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0005/000002.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0002/000002.log
/tmp/tidb_cdc_test/sink_hang/cdc_data/tmp/sorter/0007/000002.log
/tmp/tidb_cdc_test/sink_hang/tikv_down/db/000005.log
/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log
/tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log
/tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log
/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy/db/000005.log
/tmp/tidb_cdc_test/sink_hang/tikv_down.log
/tmp/tidb_cdc_test/big_txn/tikv_down/db/000005.log
/tmp/tidb_cdc_test/sink_hang/tikv3.log
/tmp/tidb_cdc_test/sink_hang/stdout.log
/tmp/tidb_cdc_test/sink_hang/sync_diff/output/sync_diff.log
/tmp/tidb_cdc_test/sink_hang/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/sink_hang/pd1.log
/tmp/tidb_cdc_test/sink_hang/tidb_other.log
/tmp/tidb_cdc_test/sink_hang/sync_diff_inspector.log
/tmp/tidb_cdc_test/sink_hang/tikv1.log
/tmp/tidb_cdc_test/sink_hang/tikv3/db/000005.log
/tmp/tidb_cdc_test/sink_hang/tidb_down.log
/tmp/tidb_cdc_test/sink_hang/tikv2.log
/tmp/tidb_cdc_test/sink_hang/pd1/region-meta/000001.log
/tmp/tidb_cdc_test/sink_hang/pd1/hot-region/000001.log
/tmp/tidb_cdc_test/sink_hang/tidb-slow.log
/tmp/tidb_cdc_test/sink_hang/tikv2/db/000005.log
/tmp/tidb_cdc_test/sink_hang/down_pd.log
/tmp/tidb_cdc_test/sink_hang/down_pd/region-meta/000001.log
/tmp/tidb_cdc_test/sink_hang/down_pd/hot-region/000001.log
/tmp/tidb_cdc_test/sink_hang/cdc.log
/tmp/tidb_cdc_test/sink_hang/tikv1/db/000005.log
+ ls -alh log-G16.tar.gz
-rw-r--r-- 1 jenkins jenkins 12M Apr 28 11:56 log-G16.tar.gz
[Pipeline] archiveArtifacts
Archiving artifacts
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
Starting Upstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
Recording fingerprints
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME	VARIABLE_VALUE	COMMENT
bootstrapped	True	Bootstrap flag. Do not delete.
tidb_server_version	179	Bootstrap version. Do not delete.
system_tz	Asia/Shanghai	TiDB Global System Timezone.
new_collation_enabled	True	If the new collations are enabled. Do not edit it.
ddl_table_version	3	DDL Table Version. Do not delete.
tikv_gc_leader_uuid	63c8b6468480013	Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc	host:xtiflow-release-7-5-pull-cdc-integration-storage-test-293-60036, pid:9847, start at 2024-04-28 11:56:44.476787388 +0800 CST m=+5.398152758	Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease	20240428-11:58:44.484 +0800	Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency	true	Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable	true	Current GC enable status
tikv_gc_run_interval	10m0s	GC run interval, at least 10m, in Go format.
tikv_gc_life_time	10m0s	All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time	20240428-11:56:44.484 +0800	The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point	20240428-11:46:44.484 +0800	All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME	VARIABLE_VALUE	COMMENT
bootstrapped	True	Bootstrap flag. Do not delete.
tidb_server_version	179	Bootstrap version. Do not delete.
system_tz	Asia/Shanghai	TiDB Global System Timezone.
new_collation_enabled	True	If the new collations are enabled. Do not edit it.
ddl_table_version	3	DDL Table Version. Do not delete.
tikv_gc_leader_uuid	63c8b6468480013	Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc	host:xtiflow-release-7-5-pull-cdc-integration-storage-test-293-60036, pid:9847, start at 2024-04-28 11:56:44.476787388 +0800 CST m=+5.398152758	Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease	20240428-11:58:44.484 +0800	Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency	true	Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable	true	Current GC enable status
tikv_gc_run_interval	10m0s	GC run interval, at least 10m, in Go format.
tikv_gc_life_time	10m0s	All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time	20240428-11:56:44.484 +0800	The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point	20240428-11:46:44.484 +0800	All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME	VARIABLE_VALUE	COMMENT
bootstrapped	True	Bootstrap flag. Do not delete.
tidb_server_version	179	Bootstrap version. Do not delete.
system_tz	Asia/Shanghai	TiDB Global System Timezone.
new_collation_enabled	True	If the new collations are enabled. Do not edit it.
ddl_table_version	3	DDL Table Version. Do not delete.
tikv_gc_leader_uuid	63c8b646acc0014	Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc	host:xtiflow-release-7-5-pull-cdc-integration-storage-test-293-60036, pid:9922, start at 2024-04-28 11:56:44.636018708 +0800 CST m=+5.482398223	Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease	20240428-11:58:44.644 +0800	Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency	true	Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable	true	Current GC enable status
tikv_gc_run_interval	10m0s	GC run interval, at least 10m, in Go format.
tikv_gc_life_time	10m0s	All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time	20240428-11:56:44.645 +0800	The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point	20240428-11:46:44.645 +0800	All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition:         Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch:      HEAD
UTC Build Time:  2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile:         RELWITHDEBINFO

Raft Proxy
Git Commit Hash:   521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time:    2024-04-18 07:28:40
Rust Version:      rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine:    tiflash
Prometheus Prefix: tiflash_proxy_
Profile:           release
Enable Features:    portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log
arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n    TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
/tmp/tidb_cdc_test/big_txn/tikv_down/db/000026.log
/tmp/tidb_cdc_test/big_txn/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/big_txn/tidb_down.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0004/000002.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0007/000002.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000007.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000016.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000006.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000011.log
VARIABLE_NAME	VARIABLE_VALUE	COMMENT
bootstrapped	True	Bootstrap flag. Do not delete.
tidb_server_version	179	Bootstrap version. Do not delete.
system_tz	Asia/Shanghai	TiDB Global System Timezone.
new_collation_enabled	True	If the new collations are enabled. Do not edit it.
ddl_table_version	3	DDL Table Version. Do not delete.
tikv_gc_leader_uuid	63c8b6490f00009	Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc	host:xtiflow-release-7-5-pull-cdc-integration-storage-test-293-mks8q, pid:16731, start at 2024-04-28 11:56:47.045473036 +0800 CST m=+5.272949518	Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease	20240428-11:58:47.055 +0800	Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency	true	Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable	true	Current GC enable status
tikv_gc_run_interval	10m0s	GC run interval, at least 10m, in Go format.
tikv_gc_life_time	10m0s	All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time	20240428-11:56:47.036 +0800	The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point	20240428-11:46:47.036 +0800	All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000008.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0005/000005.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0006/000002.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0002/000002.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000016.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000014.log
***************** properties *****************
"recordcount"="20"
"readallfields"="true"
"dotransactions"="false"
"workload"="core"
"readproportion"="0"
"insertproportion"="0"
"operationcount"="0"
"mysql.db"="changefeed_auto_stop_1"
"updateproportion"="0"
"scanproportion"="0"
"requestdistribution"="uniform"
"mysql.port"="4000"
"mysql.user"="root"
"threadcount"="4"
"mysql.host"="127.0.0.1"
**********************************************
Run finished, takes 24.414252ms
INSERT - Takes(s): 0.0, Count: 17, OPS: 3227.2, Avg(us): 2186, Min(us): 946, Max(us): 18910, 95th(us): 19000, 99th(us): 19000
***************** properties *****************
"mysql.db"="changefeed_auto_stop_2"
"threadcount"="4"
"updateproportion"="0"
"recordcount"="20"
"readallfields"="true"
"requestdistribution"="uniform"
"dotransactions"="false"
"readproportion"="0"
"scanproportion"="0"
"workload"="core"
"mysql.port"="4000"
"insertproportion"="0"
"operationcount"="0"
"mysql.host"="127.0.0.1"
"mysql.user"="root"
**********************************************
Run finished, takes 8.672018ms
INSERT - Takes(s): 0.0, Count: 20, OPS: 4512.5, Avg(us): 1649, Min(us): 895, Max(us): 4264, 95th(us): 5000, 99th(us): 5000
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000018.log
VARIABLE_NAME	VARIABLE_VALUE	COMMENT
bootstrapped	True	Bootstrap flag. Do not delete.
tidb_server_version	179	Bootstrap version. Do not delete.
system_tz	Asia/Shanghai	TiDB Global System Timezone.
new_collation_enabled	True	If the new collations are enabled. Do not edit it.
ddl_table_version	3	DDL Table Version. Do not delete.
tikv_gc_leader_uuid	63c8b6490f00009	Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc	host:xtiflow-release-7-5-pull-cdc-integration-storage-test-293-mks8q, pid:16731, start at 2024-04-28 11:56:47.045473036 +0800 CST m=+5.272949518	Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease	20240428-11:58:47.055 +0800	Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency	true	Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable	true	Current GC enable status
tikv_gc_run_interval	10m0s	GC run interval, at least 10m, in Go format.
tikv_gc_life_time	10m0s	All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time	20240428-11:56:47.036 +0800	The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point	20240428-11:46:47.036 +0800	All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME	VARIABLE_VALUE	COMMENT
bootstrapped	True	Bootstrap flag. Do not delete.
tidb_server_version	179	Bootstrap version. Do not delete.
system_tz	Asia/Shanghai	TiDB Global System Timezone.
new_collation_enabled	True	If the new collations are enabled. Do not edit it.
ddl_table_version	3	DDL Table Version. Do not delete.
tikv_gc_leader_uuid	63c8b6490d4000f	Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc	host:xtiflow-release-7-5-pull-cdc-integration-storage-test-293-mks8q, pid:16811, start at 2024-04-28 11:56:47.045265225 +0800 CST m=+5.206579122	Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease	20240428-11:58:47.055 +0800	Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency	true	Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable	true	Current GC enable status
tikv_gc_run_interval	10m0s	GC run interval, at least 10m, in Go format.
tikv_gc_life_time	10m0s	All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time	20240428-11:56:47.029 +0800	The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point	20240428-11:46:47.029 +0800	All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition:         Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch:      HEAD
UTC Build Time:  2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile:         RELWITHDEBINFO

Raft Proxy
Git Commit Hash:   521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time:    2024-04-18 07:28:40
Rust Version:      rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine:    tiflash
Prometheus Prefix: tiflash_proxy_
Profile:           release
Enable Features:    portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log
arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n    TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
***************** properties *****************
"scanproportion"="0"
"threadcount"="4"
"workload"="core"
"operationcount"="0"
"dotransactions"="false"
"readproportion"="0"
"readallfields"="true"
"mysql.host"="127.0.0.1"
"updateproportion"="0"
"requestdistribution"="uniform"
"mysql.port"="4000"
"mysql.db"="changefeed_auto_stop_3"
"mysql.user"="root"
"recordcount"="20"
"insertproportion"="0"
**********************************************
Run finished, takes 9.939625ms
INSERT - Takes(s): 0.0, Count: 20, OPS: 3946.0, Avg(us): 1872, Min(us): 1011, Max(us): 4869, 95th(us): 5000, 99th(us): 5000
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000010.log
***************** properties *****************
"mysql.db"="changefeed_auto_stop_4"
"mysql.host"="127.0.0.1"
"mysql.port"="4000"
"operationcount"="0"
"updateproportion"="0"
"requestdistribution"="uniform"
"workload"="core"
"readallfields"="true"
"dotransactions"="false"
"mysql.user"="root"
"readproportion"="0"
"threadcount"="4"
"scanproportion"="0"
"insertproportion"="0"
"recordcount"="20"
**********************************************
Run finished, takes 10.04408ms
INSERT - Takes(s): 0.0, Count: 20, OPS: 3736.7, Avg(us): 1907, Min(us): 1014, Max(us): 4725, 95th(us): 5000, 99th(us): 5000
[Sun Apr 28 11:56:50 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info'
+ GO_FAILPOINTS=
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1130911311.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info
* About to connect() to 127.0.0.1 port 8301 (#0)
*   Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8301; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000008.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0001/000012.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0003/000002.log
/tmp/tidb_cdc_test/big_txn/cdc_data/tmp/sorter/0000/000002.log
/tmp/tidb_cdc_test/big_txn/tiflash/log/error.log
/tmp/tidb_cdc_test/big_txn/tiflash/log/server.log
/tmp/tidb_cdc_test/big_txn/tiflash/log/proxy.log
/tmp/tidb_cdc_test/big_txn/tiflash/db/proxy/db/000005.log
/tmp/tidb_cdc_test/big_txn/tikv3/db/000005.log
+ cd /tmp/tidb_cdc_test/synced_status
++ run_cdc_cli_tso_query 127.0.0.1 2379
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18067.out cli tso query --pd=http://127.0.0.1:2379
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info
* About to connect() to 127.0.0.1 port 8301 (#0)
*   Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8301
> Accept: */*
> 
< HTTP/1.1 200 OK
< Date: Sun, 28 Apr 2024 03:56:53 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
< 
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/a2010534-a11b-4fa6-803c-861b2db3b2d0
	{"id":"a2010534-a11b-4fa6-803c-861b2db3b2d0","address":"127.0.0.1:8301","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef443d
	a2010534-a11b-4fa6-803c-861b2db3b2d0

/tidb/cdc/default/default/upstream/7362761915315432663
	{"id":7362761915315432663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/a2010534-a11b-4fa6-803c-861b2db3b2d0
	{"id":"a2010534-a11b-4fa6-803c-861b2db3b2d0","address":"127.0.0.1:8301","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef443d
	a2010534-a11b-4fa6-803c-861b2db3b2d0

/tidb/cdc/default/default/upstream/7362761915315432663
	{"id":7362761915315432663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/a2010534-a11b-4fa6-803c-861b2db3b2d0
	{"id":"a2010534-a11b-4fa6-803c-861b2db3b2d0","address":"127.0.0.1:8301","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef443d
	a2010534-a11b-4fa6-803c-861b2db3b2d0

/tidb/cdc/default/default/upstream/7362761915315432663
	{"id":7362761915315432663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
[Sun Apr 28 11:56:53 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info'
+ [[ no != \n\o ]]
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)'
+ (( i = 0 ))
+ (( i <= 50 ))
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1136111363.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379
++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info
* About to connect() to 127.0.0.1 port 8302 (#0)
*   Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8302; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ set +x
+ tso='449387328290160641
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449387328290160641 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
+ start_ts=449387328290160641
+ run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test
[Sun Apr 28 11:56:53 CST 2024] <<<<<< START cdc server in synced_status case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ GO_FAILPOINTS=
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1810518107.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
*   Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
/tmp/tidb_cdc_test/big_txn/tikv3/db/000026.log
/tmp/tidb_cdc_test/big_txn/tidb.log
/tmp/tidb_cdc_test/big_txn/tikv2.log
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info
* About to connect() to 127.0.0.1 port 8302 (#0)
*   Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8302
> Accept: */*
> 
< HTTP/1.1 200 OK
< Date: Sun, 28 Apr 2024 03:56:56 GMT
< Content-Length: 867
< Content-Type: text/plain; charset=utf-8
< 
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='

*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/24c23a07-23af-421b-a6eb-d997de241c00
	{"id":"24c23a07-23af-421b-a6eb-d997de241c00","address":"127.0.0.1:8302","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/capture/a2010534-a11b-4fa6-803c-861b2db3b2d0
	{"id":"a2010534-a11b-4fa6-803c-861b2db3b2d0","address":"127.0.0.1:8301","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef443d
	a2010534-a11b-4fa6-803c-861b2db3b2d0

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef4464
	24c23a07-23af-421b-a6eb-d997de241c00

/tidb/cdc/default/default/upstream/7362761915315432663
	{"id":7362761915315432663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '

*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/24c23a07-23af-421b-a6eb-d997de241c00
	{"id":"24c23a07-23af-421b-a6eb-d997de241c00","address":"127.0.0.1:8302","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/capture/a2010534-a11b-4fa6-803c-861b2db3b2d0
	{"id":"a2010534-a11b-4fa6-803c-861b2db3b2d0","address":"127.0.0.1:8301","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef443d
	a2010534-a11b-4fa6-803c-861b2db3b2d0

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef4464
	24c23a07-23af-421b-a6eb-d997de241c00

/tidb/cdc/default/default/upstream/7362761915315432663
	{"id":7362761915315432663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '

*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/24c23a07-23af-421b-a6eb-d997de241c00
	{"id":"24c23a07-23af-421b-a6eb-d997de241c00","address":"127.0.0.1:8302","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/capture/a2010534-a11b-4fa6-803c-861b2db3b2d0
	{"id":"a2010534-a11b-4fa6-803c-861b2db3b2d0","address":"127.0.0.1:8301","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef443d
	a2010534-a11b-4fa6-803c-861b2db3b2d0

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8ef4464
	24c23a07-23af-421b-a6eb-d997de241c00

/tidb/cdc/default/default/upstream/7362761915315432663
	{"id":7362761915315432663,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ workdir=/tmp/tidb_cdc_test/changefeed_auto_stop
+ sink_uri='file:///tmp/tidb_cdc_test/changefeed_auto_stop/storage_test/ticdc-changefeed-auto-stop-test-26692?protocol=canal-json&enable-tidb-extension=true'
+ consumer_replica_config=
+ log_suffix=
++ pwd
+ pwd=/tmp/tidb_cdc_test/changefeed_auto_stop
++ date
+ echo '[Sun Apr 28 11:56:56 CST 2024] <<<<<< START storage consumer in changefeed_auto_stop case >>>>>>'
[Sun Apr 28 11:56:56 CST 2024] <<<<<< START storage consumer in changefeed_auto_stop case >>>>>>
+ cd /tmp/tidb_cdc_test/changefeed_auto_stop
+ '[' '' '!=' '' ']'
+ cd /tmp/tidb_cdc_test/changefeed_auto_stop
+ set +x
+ cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_auto_stop/storage_test/ticdc-changefeed-auto-stop-test-26692?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false'
check_changefeed_state http://127.0.0.1:2379 3ba0fa1b-4f27-49c7-8437-2be61e4f83f6 normal null
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=3ba0fa1b-4f27-49c7-8437-2be61e4f83f6
+ expected_state=normal
+ error_msg=null
+ tls_dir=null
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 3ba0fa1b-4f27-49c7-8437-2be61e4f83f6 -s
+ info='{
  "upstream_id": 7362761915315432663,
  "namespace": "default",
  "id": "3ba0fa1b-4f27-49c7-8437-2be61e4f83f6",
  "state": "normal",
  "checkpoint_tso": 449387327529943041,
  "checkpoint_time": "2024-04-28 11:56:49.535",
  "error": null
}'
+ echo '{
  "upstream_id": 7362761915315432663,
  "namespace": "default",
  "id": "3ba0fa1b-4f27-49c7-8437-2be61e4f83f6",
  "state": "normal",
  "checkpoint_tso": 449387327529943041,
  "checkpoint_time": "2024-04-28 11:56:49.535",
  "error": null
}'
{
  "upstream_id": 7362761915315432663,
  "namespace": "default",
  "id": "3ba0fa1b-4f27-49c7-8437-2be61e4f83f6",
  "state": "normal",
  "checkpoint_tso": 449387327529943041,
  "checkpoint_time": "2024-04-28 11:56:49.535",
  "error": null
}
++ echo '{' '"upstream_id":' 7362761915315432663, '"namespace":' '"default",' '"id":' '"3ba0fa1b-4f27-49c7-8437-2be61e4f83f6",' '"state":' '"normal",' '"checkpoint_tso":' 449387327529943041, '"checkpoint_time":' '"2024-04-28' '11:56:49.535",' '"error":' null '}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
++ echo '{' '"upstream_id":' 7362761915315432663, '"namespace":' '"default",' '"id":' '"3ba0fa1b-4f27-49c7-8437-2be61e4f83f6",' '"state":' '"normal",' '"checkpoint_tso":' 449387327529943041, '"checkpoint_time":' '"2024-04-28' '11:56:49.535",' '"error":' null '}'
++ jq -r .error.message
+ message=null
+ [[ ! null =~ null ]]
run task successfully
table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
*   Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
> 
< HTTP/1.1 200 OK
< Date: Sun, 28 Apr 2024 03:56:56 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
< 
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/51979971-f108-4d4e-aa05-314a3c2c8f29
	{"id":"51979971-f108-4d4e-aa05-314a3c2c8f29","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8fbc6f5
	51979971-f108-4d4e-aa05-314a3c2c8f29

/tidb/cdc/default/default/upstream/7362761928687516344
	{"id":7362761928687516344,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/51979971-f108-4d4e-aa05-314a3c2c8f29
	{"id":"51979971-f108-4d4e-aa05-314a3c2c8f29","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8fbc6f5
	51979971-f108-4d4e-aa05-314a3c2c8f29

/tidb/cdc/default/default/upstream/7362761928687516344
	{"id":7362761928687516344,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '

*** owner info ***:



*** processors info ***:



*** etcd info ***:

/tidb/cdc/default/__cdc_meta__/capture/51979971-f108-4d4e-aa05-314a3c2c8f29
	{"id":"51979971-f108-4d4e-aa05-314a3c2c8f29","address":"127.0.0.1:8300","version":"v7.5.1-49-g234b58dc1"}

/tidb/cdc/default/__cdc_meta__/meta/meta-version
	1

/tidb/cdc/default/__cdc_meta__/owner/22318f22d8fbc6f5
	51979971-f108-4d4e-aa05-314a3c2c8f29

/tidb/cdc/default/default/upstream/7362761928687516344
	{"id":7362761928687516344,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ config_path=conf/changefeed.toml
+ SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1'
+ run_cdc_cli changefeed create --start-ts=449387328290160641 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18152.out cli changefeed create --start-ts=449387328290160641 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml
Create changefeed successfully!
ID: test-1
Info: {"upstream_id":7362761928687516344,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-04-28T11:56:57.314277155+08:00","start_ts":449387328290160641,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-49-g234b58dc1","resolved_ts":449387328290160641,"checkpoint_ts":449387328290160641,"checkpoint_time":"2024-04-28 11:56:52.435"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
/tmp/tidb_cdc_test/big_txn/down_pd/region-meta/000001.log
/tmp/tidb_cdc_test/big_txn/down_pd/hot-region/000001.log
/tmp/tidb_cdc_test/big_txn/cdc_storage_consumer.log

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   221  100   221    0     0   2218      0 --:--:-- --:--:-- --:--:--  2232
+ synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-28 11:56:52.435","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-04-28 11:56:58.000","info":"Data syncing is finished"}'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-28' '11:56:52.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-28' '11:56:58.000","info":"Data' syncing is 'finished"}'
++ jq .synced
+ status=true
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-28' '11:56:52.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-28' '11:56:58.000","info":"Data' syncing is 'finished"}'
++ jq -r .sink_checkpoint_ts
+ sink_checkpoint_ts='2024-04-28 11:56:52.435'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-28' '11:56:52.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-28' '11:56:58.000","info":"Data' syncing is 'finished"}'
++ jq -r .puller_resolved_ts
+ puller_resolved_ts='1970-01-01 08:00:00.000'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-28' '11:56:52.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-04-28' '11:56:58.000","info":"Data' syncing is 'finished"}'
++ jq -r .last_synced_ts
+ last_synced_ts='1970-01-01 08:00:00.000'
+ '[' true '!=' true ']'
+ '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']'
+ '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']'
++ date '+%Y-%m-%d %H:%M:%S'
+ current='2024-04-28 11:56:58'
+ echo 'sink_checkpoint_ts is 2024-04-28' 11:56:52.435
sink_checkpoint_ts is 2024-04-28 11:56:52.435
++ date -d '2024-04-28 11:56:52.435' +%s
+ checkpoint_timestamp=1714276612
++ date -d '2024-04-28 11:56:58' +%s
+ current_timestamp=1714276618
+ '[' 6 -gt 300 ']'
+ run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);'
+ check_table_exists test.t1 127.0.0.1 3306
table test.t1 not exists for 1-th check, retry later
table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later
table changefeed_auto_stop_1.usertable not exists for 3-th check, retry later
table test.t1 exists
+ sleep 5
table changefeed_auto_stop_1.usertable not exists for 4-th check, retry later
table changefeed_auto_stop_1.usertable not exists for 5-th check, retry later
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   243  100   243    0     0   3186      0 --:--:-- --:--:-- --:--:--  3197
+ synced_status='{"synced":false,"sink_checkpoint_ts":"2024-04-28 11:57:04.836","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-04-28 11:56:59.036","now_ts":"2024-04-28 11:57:06.000","info":"The data syncing is not finished, please wait"}'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-28' '11:57:04.836","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-28' '11:56:59.036","now_ts":"2024-04-28' '11:57:06.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq .synced
+ status=false
+ '[' false '!=' false ']'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-04-28' '11:57:04.836","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-04-28' '11:56:59.036","now_ts":"2024-04-28' '11:57:06.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq -r .info
+ info='The data syncing is not finished, please wait'
+ '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']'
+ sleep 130
table changefeed_auto_stop_1.usertable exists
table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later
table changefeed_auto_stop_2.usertable exists
table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later
/tmp/tidb_cdc_test/big_txn/tikv1/db/000005.log
/tmp/tidb_cdc_test/big_txn/pd1.log
/tmp/tidb_cdc_test/big_txn/sync_diff/output/sync_diff.log
/tmp/tidb_cdc_test/big_txn/tidb_other.log
/tmp/tidb_cdc_test/big_txn/tikv2/db/000034.log
table changefeed_auto_stop_3.usertable not exists for 2-th check, retry later
/tmp/tidb_cdc_test/big_txn/tikv2/db/000046.log
/tmp/tidb_cdc_test/big_txn/tikv2/db/000030.log
/tmp/tidb_cdc_test/big_txn/tikv2/db/000052.log
/tmp/tidb_cdc_test/big_txn/tikv2/db/000020.log
/tmp/tidb_cdc_test/big_txn/tikv2/db/000015.log
/tmp/tidb_cdc_test/big_txn/tikv2/db/000005.log
table changefeed_auto_stop_3.usertable not exists for 3-th check, retry later
table changefeed_auto_stop_3.usertable not exists for 4-th check, retry later
/tmp/tidb_cdc_test/big_txn/tikv2/db/000042.log
/tmp/tidb_cdc_test/big_txn/tikv_down-2024-04-28T11-54-13.571.log
table changefeed_auto_stop_3.usertable not exists for 5-th check, retry later
table changefeed_auto_stop_3.usertable exists
table changefeed_auto_stop_4.usertable exists
check diff failed 1-th time, retry later
check diff failed 2-th time, retry later
check diff failed 3-th time, retry later
check diff failed 4-th time, retry later
check diff successfully
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Sun Apr 28 11:57:29 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>>
\033[0;36m<<< Run all test success >>>\033[0m
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_storage_test-293/tiflow-cdc already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
Post stage
[Pipeline] sh
+ ls /tmp/tidb_cdc_test/
changefeed_auto_stop
cov.changefeed_auto_stop.1130911311.out
cov.changefeed_auto_stop.1136111363.out
cov.ddl_only_block_related_table.78867888.out
cov.ddl_only_block_related_table.80598061.out
cov.ddl_only_block_related_table.86578659.out
cov.ddl_only_block_related_table.cli.7939.out
cov.ddl_puller_lag.51825184.out
cov.ddl_puller_lag.cli.5133.out
cov.ddl_puller_lag.cli.5227.out
cov.foreign_key.25902592.out
cov.foreign_key.cli.2560.out
cov.foreign_key.cli.2638.out
ddl_only_block_related_table
ddl_puller_lag
foreign_key
sql_res.changefeed_auto_stop.txt
sql_res.ddl_only_block_related_table.txt
sql_res.ddl_puller_lag.txt
sql_res.foreign_key.txt
++ find /tmp/tidb_cdc_test/ -type f -name '*.log'
+ tar -cvzf log-G04.tar.gz /tmp/tidb_cdc_test/ddl_only_block_related_table/down_pd.log /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_storage_consumer.log /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/ddl_only_block_related_table/pd1.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tidb-slow.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tikv_down.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tidb.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tikv3.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tikv2.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tidb_other.log /tmp/tidb_cdc_test/ddl_only_block_related_table/stdout.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tidb_down.log /tmp/tidb_cdc_test/ddl_only_block_related_table/tikv1.log /tmp/tidb_cdc_test/ddl_only_block_related_table/sync_diff_inspector.log /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log /tmp/tidb_cdc_test/changefeed_auto_stop/down_pd.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_storage_consumer.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv3/db/000005.log /tmp/tidb_cdc_test/changefeed_auto_stop/stdout2.log /tmp/tidb_cdc_test/changefeed_auto_stop/pd1.log /tmp/tidb_cdc_test/changefeed_auto_stop/tidb-slow.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv_down.log /tmp/tidb_cdc_test/changefeed_auto_stop/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/changefeed_auto_stop/stdout1.log /tmp/tidb_cdc_test/changefeed_auto_stop/tidb.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv3.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv2.log /tmp/tidb_cdc_test/changefeed_auto_stop/tidb_other.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/changefeed_auto_stop/tidb_down.log /tmp/tidb_cdc_test/changefeed_auto_stop/pd1/region-meta/000001.log /tmp/tidb_cdc_test/changefeed_auto_stop/pd1/hot-region/000001.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv1.log /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv1/db/000005.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv2/db/000005.log /tmp/tidb_cdc_test/changefeed_auto_stop/sync_diff_inspector.log /tmp/tidb_cdc_test/changefeed_auto_stop/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/changefeed_auto_stop/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/changefeed_auto_stop/tikv_down/db/000005.log /tmp/tidb_cdc_test/ddl_puller_lag/down_pd.log /tmp/tidb_cdc_test/ddl_puller_lag/cdc_storage_consumer.log /tmp/tidb_cdc_test/ddl_puller_lag/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/ddl_puller_lag/pd1.log /tmp/tidb_cdc_test/ddl_puller_lag/tidb-slow.log /tmp/tidb_cdc_test/ddl_puller_lag/tikv_down.log /tmp/tidb_cdc_test/ddl_puller_lag/tidb.log /tmp/tidb_cdc_test/ddl_puller_lag/tikv3.log /tmp/tidb_cdc_test/ddl_puller_lag/tikv2.log /tmp/tidb_cdc_test/ddl_puller_lag/tidb_other.log /tmp/tidb_cdc_test/ddl_puller_lag/stdout.log /tmp/tidb_cdc_test/ddl_puller_lag/tidb_down.log /tmp/tidb_cdc_test/ddl_puller_lag/tikv1.log /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log /tmp/tidb_cdc_test/foreign_key/down_pd.log /tmp/tidb_cdc_test/foreign_key/cdc_storage_consumer.log /tmp/tidb_cdc_test/foreign_key/cdc_storage_consumer_stdout.log /tmp/tidb_cdc_test/foreign_key/pd1.log /tmp/tidb_cdc_test/foreign_key/tidb-slow.log /tmp/tidb_cdc_test/foreign_key/tikv_down.log /tmp/tidb_cdc_test/foreign_key/tidb.log /tmp/tidb_cdc_test/foreign_key/tikv3.log /tmp/tidb_cdc_test/foreign_key/tikv2.log /tmp/tidb_cdc_test/foreign_key/tidb_other.log /tmp/tidb_cdc_test/foreign_key/stdout.log /tmp/tidb_cdc_test/foreign_key/tidb_down.log /tmp/tidb_cdc_test/foreign_key/tikv1.log /tmp/tidb_cdc_test/foreign_key/sync_diff_inspector.log /tmp/tidb_cdc_test/foreign_key/cdc.log
tar: Removing leading `/' from member names
/tmp/tidb_cdc_test/ddl_only_block_related_table/down_pd.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_storage_consumer.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/pd1.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tidb-slow.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tikv_down.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tidb.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tikv3.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tikv2.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tidb_other.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/stdout.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tidb_down.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/tikv1.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/sync_diff_inspector.log
/tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log
/tmp/tidb_cdc_test/changefeed_auto_stop/down_pd.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0000/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0002/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0005/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0001/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0007/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0003/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0006/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1/tmp/sorter/0004/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_storage_consumer.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv3/db/000005.log
/tmp/tidb_cdc_test/changefeed_auto_stop/stdout2.log
/tmp/tidb_cdc_test/changefeed_auto_stop/pd1.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tidb-slow.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv_down.log
/tmp/tidb_cdc_test/changefeed_auto_stop/sync_diff/output/sync_diff.log
/tmp/tidb_cdc_test/changefeed_auto_stop/stdout1.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tidb.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv3.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv2.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tidb_other.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0000/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0002/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0005/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0001/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0007/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0003/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0006/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2/tmp/sorter/0004/000002.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tidb_down.log
/tmp/tidb_cdc_test/changefeed_auto_stop/pd1/region-meta/000001.log
/tmp/tidb_cdc_test/changefeed_auto_stop/pd1/hot-region/000001.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv1.log
/tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy/db/000005.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv1/db/000005.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv2/db/000005.log
/tmp/tidb_cdc_test/changefeed_auto_stop/sync_diff_inspector.log
/tmp/tidb_cdc_test/changefeed_auto_stop/down_pd/region-meta/000001.log
/tmp/tidb_cdc_test/changefeed_auto_stop/down_pd/hot-region/000001.log
/tmp/tidb_cdc_test/changefeed_auto_stop/tikv_down/db/000005.log
/tmp/tidb_cdc_test/ddl_puller_lag/down_pd.log
/tmp/tidb_cdc_test/ddl_puller_lag/cdc_storage_consumer.log
/tmp/tidb_cdc_test/ddl_puller_lag/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/ddl_puller_lag/pd1.log
/tmp/tidb_cdc_test/ddl_puller_lag/tidb-slow.log
/tmp/tidb_cdc_test/ddl_puller_lag/tikv_down.log
/tmp/tidb_cdc_test/ddl_puller_lag/tidb.log
/tmp/tidb_cdc_test/ddl_puller_lag/tikv3.log
/tmp/tidb_cdc_test/ddl_puller_lag/tikv2.log
/tmp/tidb_cdc_test/ddl_puller_lag/tidb_other.log
/tmp/tidb_cdc_test/ddl_puller_lag/stdout.log
/tmp/tidb_cdc_test/ddl_puller_lag/tidb_down.log
/tmp/tidb_cdc_test/ddl_puller_lag/tikv1.log
/tmp/tidb_cdc_test/ddl_puller_lag/cdc.log
/tmp/tidb_cdc_test/foreign_key/down_pd.log
/tmp/tidb_cdc_test/foreign_key/cdc_storage_consumer.log
/tmp/tidb_cdc_test/foreign_key/cdc_storage_consumer_stdout.log
/tmp/tidb_cdc_test/foreign_key/pd1.log
/tmp/tidb_cdc_test/foreign_key/tidb-slow.log
/tmp/tidb_cdc_test/foreign_key/tikv_down.log
/tmp/tidb_cdc_test/foreign_key/tidb.log
/tmp/tidb_cdc_test/foreign_key/tikv3.log
/tmp/tidb_cdc_test/foreign_key/tikv2.log
/tmp/tidb_cdc_test/foreign_key/tidb_other.log
/tmp/tidb_cdc_test/foreign_key/stdout.log
/tmp/tidb_cdc_test/foreign_key/tidb_down.log
/tmp/tidb_cdc_test/foreign_key/tikv1.log
/tmp/tidb_cdc_test/foreign_key/sync_diff_inspector.log
/tmp/tidb_cdc_test/foreign_key/cdc.log
+ ls -alh log-G04.tar.gz
-rw-r--r-- 1 jenkins jenkins 13M Apr 28 11:57 log-G04.tar.gz
[Pipeline] archiveArtifacts
Archiving artifacts
Recording fingerprints
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
/tmp/tidb_cdc_test/big_txn/tidb-slow.log
/tmp/tidb_cdc_test/big_txn/tikv3.log
/tmp/tidb_cdc_test/big_txn/tikv2-2024-04-28T11-54-15.055.log
/tmp/tidb_cdc_test/big_txn/sync_diff_inspector.log
/tmp/tidb_cdc_test/big_txn/tikv_down.log
+ ls -alh log-G14.tar.gz
-rw-r--r-- 1 jenkins jenkins 691M Apr 28 11:58 log-G14.tar.gz
[Pipeline] archiveArtifacts
Archiving artifacts
Recording fingerprints
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G14'
Sending interrupt signal to process
Killing processes
kill finished with exit code 0
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed

  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100   221  100   221    0     0   2666      0 --:--:-- --:--:-- --:--:--  2695
+ synced_status='{"synced":true,"sink_checkpoint_ts":"2024-04-28 11:59:15.987","puller_resolved_ts":"2024-04-28 11:59:07.986","last_synced_ts":"2024-04-28 11:56:59.036","now_ts":"2024-04-28 11:59:16.000","info":"Data syncing is finished"}'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-04-28' '11:59:15.987","puller_resolved_ts":"2024-04-28' '11:59:07.986","last_synced_ts":"2024-04-28' '11:56:59.036","now_ts":"2024-04-28' '11:59:16.000","info":"Data' syncing is 'finished"}'
++ jq .synced
+ status=true
+ '[' true '!=' true ']'
+ kill_pd
++ ps aux
++ grep pd-server
++ grep /tmp/tidb_cdc_test/synced_status
+ info='jenkins    15985  8.5  0.0 13839916 140892 ?     Sl   11:56   0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380
jenkins    16054  5.7  0.0 13896936 136920 ?     Sl   11:56   0:09 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd'
++ ps aux
++ grep pd-server
++ grep /tmp/tidb_cdc_test/synced_status
++ awk '{print $2}'
++ xargs kill -9
+ sleep 20
++ stop_tidb_cluster
/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/run.sh: line 1: 18363 Terminated              sleep 20
script returned exit code 143
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G09'
[Pipeline] // parallel
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
ERROR: script returned exit code 1
Finished: FAILURE